langchain 0.2.10 → 0.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/chat_models/universal.cjs +1 -0
- package/chat_models/universal.d.cts +1 -0
- package/chat_models/universal.d.ts +1 -0
- package/chat_models/universal.js +1 -0
- package/dist/chat_models/universal.cjs +600 -0
- package/dist/chat_models/universal.d.ts +115 -0
- package/dist/chat_models/universal.js +595 -0
- package/dist/document_loaders/web/assemblyai.cjs +9 -1
- package/dist/document_loaders/web/assemblyai.js +9 -1
- package/dist/experimental/chrome_ai/app/dist/bundle.cjs +1250 -0
- package/dist/experimental/chrome_ai/app/dist/bundle.d.ts +1 -0
- package/dist/experimental/chrome_ai/app/dist/bundle.js +1249 -0
- package/dist/load/import_constants.cjs +1 -0
- package/dist/load/import_constants.js +1 -0
- package/package.json +65 -6
- package/dist/agents/tests/agent.int.test.d.ts +0 -1
- package/dist/agents/tests/agent.int.test.js +0 -309
- package/dist/agents/tests/chat_convo_output_parser.test.d.ts +0 -1
- package/dist/agents/tests/chat_convo_output_parser.test.js +0 -91
- package/dist/agents/tests/create_openai_functions_agent.int.test.d.ts +0 -2
- package/dist/agents/tests/create_openai_functions_agent.int.test.js +0 -71
- package/dist/agents/tests/create_openai_tools_agent.int.test.d.ts +0 -1
- package/dist/agents/tests/create_openai_tools_agent.int.test.js +0 -75
- package/dist/agents/tests/create_react_agent.int.test.d.ts +0 -1
- package/dist/agents/tests/create_react_agent.int.test.js +0 -32
- package/dist/agents/tests/create_structured_chat_agent.int.test.d.ts +0 -1
- package/dist/agents/tests/create_structured_chat_agent.int.test.js +0 -32
- package/dist/agents/tests/create_tool_calling_agent.int.test.d.ts +0 -1
- package/dist/agents/tests/create_tool_calling_agent.int.test.js +0 -122
- package/dist/agents/tests/create_xml_agent.int.test.d.ts +0 -1
- package/dist/agents/tests/create_xml_agent.int.test.js +0 -32
- package/dist/agents/tests/json.test.d.ts +0 -1
- package/dist/agents/tests/json.test.js +0 -74
- package/dist/agents/tests/react.test.d.ts +0 -1
- package/dist/agents/tests/react.test.js +0 -44
- package/dist/agents/tests/runnable.int.test.d.ts +0 -1
- package/dist/agents/tests/runnable.int.test.js +0 -104
- package/dist/agents/tests/sql.test.d.ts +0 -1
- package/dist/agents/tests/sql.test.js +0 -133
- package/dist/agents/tests/structured_chat_output_parser.test.d.ts +0 -1
- package/dist/agents/tests/structured_chat_output_parser.test.js +0 -35
- package/dist/agents/tests/structured_chat_output_parser_with_retries.int.test.d.ts +0 -1
- package/dist/agents/tests/structured_chat_output_parser_with_retries.int.test.js +0 -44
- package/dist/agents/tests/structured_output_runnables.int.test.d.ts +0 -1
- package/dist/agents/tests/structured_output_runnables.int.test.js +0 -112
- package/dist/agents/toolkits/tests/conversational_retrieval.int.test.d.ts +0 -1
- package/dist/agents/toolkits/tests/conversational_retrieval.int.test.js +0 -41
- package/dist/cache/tests/file_system.int.test.d.ts +0 -1
- package/dist/cache/tests/file_system.int.test.js +0 -32
- package/dist/chains/openai_functions/tests/create_runnable_chains.int.test.d.ts +0 -1
- package/dist/chains/openai_functions/tests/create_runnable_chains.int.test.js +0 -139
- package/dist/chains/openai_functions/tests/extraction.int.test.d.ts +0 -1
- package/dist/chains/openai_functions/tests/extraction.int.test.js +0 -33
- package/dist/chains/openai_functions/tests/openapi.int.test.d.ts +0 -1
- package/dist/chains/openai_functions/tests/openapi.int.test.js +0 -130
- package/dist/chains/openai_functions/tests/openapi.test.d.ts +0 -1
- package/dist/chains/openai_functions/tests/openapi.test.js +0 -172
- package/dist/chains/openai_functions/tests/structured_output.int.test.d.ts +0 -1
- package/dist/chains/openai_functions/tests/structured_output.int.test.js +0 -40
- package/dist/chains/openai_functions/tests/structured_output.test.d.ts +0 -1
- package/dist/chains/openai_functions/tests/structured_output.test.js +0 -102
- package/dist/chains/openai_functions/tests/tagging.int.test.d.ts +0 -1
- package/dist/chains/openai_functions/tests/tagging.int.test.js +0 -21
- package/dist/chains/query_constructor/tests/query_chain.int.test.d.ts +0 -1
- package/dist/chains/query_constructor/tests/query_chain.int.test.js +0 -93
- package/dist/chains/query_constructor/tests/query_parser.test.d.ts +0 -1
- package/dist/chains/query_constructor/tests/query_parser.test.js +0 -28
- package/dist/chains/question_answering/tests/load.int.test.d.ts +0 -1
- package/dist/chains/question_answering/tests/load.int.test.js +0 -39
- package/dist/chains/router/tests/multi_prompt.int.test.d.ts +0 -1
- package/dist/chains/router/tests/multi_prompt.int.test.js +0 -45
- package/dist/chains/router/tests/multi_prompt.test.d.ts +0 -1
- package/dist/chains/router/tests/multi_prompt.test.js +0 -62
- package/dist/chains/router/tests/multi_retrieval_qa.int.test.d.ts +0 -1
- package/dist/chains/router/tests/multi_retrieval_qa.int.test.js +0 -67
- package/dist/chains/router/tests/multi_retrieval_qa.test.d.ts +0 -1
- package/dist/chains/router/tests/multi_retrieval_qa.test.js +0 -125
- package/dist/chains/summarization/tests/load.int.test.d.ts +0 -1
- package/dist/chains/summarization/tests/load.int.test.js +0 -37
- package/dist/chains/tests/api_chain.int.test.d.ts +0 -1
- package/dist/chains/tests/api_chain.int.test.js +0 -55
- package/dist/chains/tests/combine_docs_chain.int.test.d.ts +0 -1
- package/dist/chains/tests/combine_docs_chain.int.test.js +0 -50
- package/dist/chains/tests/combine_docs_chain.test.d.ts +0 -1
- package/dist/chains/tests/combine_docs_chain.test.js +0 -98
- package/dist/chains/tests/constitutional_chain.int.test.d.ts +0 -1
- package/dist/chains/tests/constitutional_chain.int.test.js +0 -30
- package/dist/chains/tests/constitutional_chain.test.d.ts +0 -1
- package/dist/chains/tests/constitutional_chain.test.js +0 -63
- package/dist/chains/tests/conversation_chain.int.test.d.ts +0 -1
- package/dist/chains/tests/conversation_chain.int.test.js +0 -9
- package/dist/chains/tests/conversational_retrieval_chain.int.test.d.ts +0 -1
- package/dist/chains/tests/conversational_retrieval_chain.int.test.js +0 -243
- package/dist/chains/tests/example_data/open_meteo_docs.d.ts +0 -1
- package/dist/chains/tests/example_data/open_meteo_docs.js +0 -29
- package/dist/chains/tests/history_aware_retriever.int.test.d.ts +0 -1
- package/dist/chains/tests/history_aware_retriever.int.test.js +0 -41
- package/dist/chains/tests/history_aware_retriever.test.d.ts +0 -1
- package/dist/chains/tests/history_aware_retriever.test.js +0 -27
- package/dist/chains/tests/llm_chain.int.test.d.ts +0 -1
- package/dist/chains/tests/llm_chain.int.test.js +0 -119
- package/dist/chains/tests/openai_moderation.int.test.d.ts +0 -1
- package/dist/chains/tests/openai_moderation.int.test.js +0 -30
- package/dist/chains/tests/retrieval_chain.int.test.d.ts +0 -1
- package/dist/chains/tests/retrieval_chain.int.test.js +0 -69
- package/dist/chains/tests/retrieval_chain.test.d.ts +0 -1
- package/dist/chains/tests/retrieval_chain.test.js +0 -36
- package/dist/chains/tests/sequential_chain.int.test.d.ts +0 -1
- package/dist/chains/tests/sequential_chain.int.test.js +0 -88
- package/dist/chains/tests/sequential_chain.test.d.ts +0 -1
- package/dist/chains/tests/sequential_chain.test.js +0 -295
- package/dist/chains/tests/simple_sequential_chain.int.test.d.ts +0 -1
- package/dist/chains/tests/simple_sequential_chain.int.test.js +0 -81
- package/dist/chains/tests/simple_sequential_chain.test.d.ts +0 -1
- package/dist/chains/tests/simple_sequential_chain.test.js +0 -128
- package/dist/chains/tests/sql_db_chain.int.test.d.ts +0 -1
- package/dist/chains/tests/sql_db_chain.int.test.js +0 -125
- package/dist/chains/tests/transform.test.d.ts +0 -1
- package/dist/chains/tests/transform.test.js +0 -12
- package/dist/chains/tests/vector_db_qa_chain.int.test.d.ts +0 -1
- package/dist/chains/tests/vector_db_qa_chain.int.test.js +0 -45
- package/dist/document_loaders/tests/assemblyai.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/assemblyai.int.test.js +0 -111
- package/dist/document_loaders/tests/chatgpt-blob.test.d.ts +0 -1
- package/dist/document_loaders/tests/chatgpt-blob.test.js +0 -30
- package/dist/document_loaders/tests/chatgpt.test.d.ts +0 -1
- package/dist/document_loaders/tests/chatgpt.test.js +0 -29
- package/dist/document_loaders/tests/cheerio.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/cheerio.int.test.js +0 -21
- package/dist/document_loaders/tests/college_confidential.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/college_confidential.int.test.js +0 -6
- package/dist/document_loaders/tests/confluence.test.d.ts +0 -1
- package/dist/document_loaders/tests/confluence.test.js +0 -52
- package/dist/document_loaders/tests/couchbase.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/couchbase.int.test.js +0 -28
- package/dist/document_loaders/tests/csv-blob.test.d.ts +0 -1
- package/dist/document_loaders/tests/csv-blob.test.js +0 -53
- package/dist/document_loaders/tests/csv.test.d.ts +0 -1
- package/dist/document_loaders/tests/csv.test.js +0 -41
- package/dist/document_loaders/tests/directory.test.d.ts +0 -1
- package/dist/document_loaders/tests/directory.test.js +0 -38
- package/dist/document_loaders/tests/docx.test.d.ts +0 -1
- package/dist/document_loaders/tests/docx.test.js +0 -11
- package/dist/document_loaders/tests/epub.test.d.ts +0 -1
- package/dist/document_loaders/tests/epub.test.js +0 -18
- package/dist/document_loaders/tests/example_data/github_api_responses.d.ts +0 -5
- package/dist/document_loaders/tests/example_data/github_api_responses.js +0 -91
- package/dist/document_loaders/tests/figma.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/figma.int.test.js +0 -13
- package/dist/document_loaders/tests/firecrawl.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/firecrawl.int.test.js +0 -30
- package/dist/document_loaders/tests/gitbook.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/gitbook.int.test.js +0 -14
- package/dist/document_loaders/tests/github.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/github.int.test.js +0 -86
- package/dist/document_loaders/tests/github.test.d.ts +0 -1
- package/dist/document_loaders/tests/github.test.js +0 -51
- package/dist/document_loaders/tests/hn.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/hn.int.test.js +0 -6
- package/dist/document_loaders/tests/imsdb.test.d.ts +0 -1
- package/dist/document_loaders/tests/imsdb.test.js +0 -6
- package/dist/document_loaders/tests/json-blob.test.d.ts +0 -1
- package/dist/document_loaders/tests/json-blob.test.js +0 -91
- package/dist/document_loaders/tests/json.test.d.ts +0 -1
- package/dist/document_loaders/tests/json.test.js +0 -69
- package/dist/document_loaders/tests/jsonl-blob.test.d.ts +0 -1
- package/dist/document_loaders/tests/jsonl-blob.test.js +0 -46
- package/dist/document_loaders/tests/jsonl.test.d.ts +0 -1
- package/dist/document_loaders/tests/jsonl.test.js +0 -15
- package/dist/document_loaders/tests/multi_file.test.d.ts +0 -1
- package/dist/document_loaders/tests/multi_file.test.js +0 -49
- package/dist/document_loaders/tests/notion.test.d.ts +0 -1
- package/dist/document_loaders/tests/notion.test.js +0 -11
- package/dist/document_loaders/tests/notionapi.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/notionapi.int.test.js +0 -80
- package/dist/document_loaders/tests/notionapi.test.d.ts +0 -1
- package/dist/document_loaders/tests/notionapi.test.js +0 -84
- package/dist/document_loaders/tests/notiondb.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/notiondb.int.test.js +0 -13
- package/dist/document_loaders/tests/obsidian.test.d.ts +0 -1
- package/dist/document_loaders/tests/obsidian.test.js +0 -119
- package/dist/document_loaders/tests/pdf-blob.test.d.ts +0 -1
- package/dist/document_loaders/tests/pdf-blob.test.js +0 -44
- package/dist/document_loaders/tests/pdf.test.d.ts +0 -1
- package/dist/document_loaders/tests/pdf.test.js +0 -25
- package/dist/document_loaders/tests/playwright_web.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/playwright_web.int.test.js +0 -27
- package/dist/document_loaders/tests/pptx.test.d.ts +0 -1
- package/dist/document_loaders/tests/pptx.test.js +0 -17
- package/dist/document_loaders/tests/puppeteer.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/puppeteer.int.test.js +0 -47
- package/dist/document_loaders/tests/recursive_url.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/recursive_url.int.test.js +0 -64
- package/dist/document_loaders/tests/s3.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/s3.int.test.js +0 -48
- package/dist/document_loaders/tests/searchapi.test.d.ts +0 -1
- package/dist/document_loaders/tests/searchapi.test.js +0 -29
- package/dist/document_loaders/tests/serpapi.test.d.ts +0 -1
- package/dist/document_loaders/tests/serpapi.test.js +0 -21
- package/dist/document_loaders/tests/sitemap.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/sitemap.int.test.js +0 -28
- package/dist/document_loaders/tests/sonix_audio.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/sonix_audio.int.test.js +0 -55
- package/dist/document_loaders/tests/sort_xyz_blockchain.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/sort_xyz_blockchain.int.test.js +0 -38
- package/dist/document_loaders/tests/srt-blob.test.d.ts +0 -1
- package/dist/document_loaders/tests/srt-blob.test.js +0 -18
- package/dist/document_loaders/tests/srt.test.d.ts +0 -1
- package/dist/document_loaders/tests/srt.test.js +0 -16
- package/dist/document_loaders/tests/text-blob.test.d.ts +0 -1
- package/dist/document_loaders/tests/text-blob.test.js +0 -14
- package/dist/document_loaders/tests/text.test.d.ts +0 -1
- package/dist/document_loaders/tests/text.test.js +0 -22
- package/dist/document_loaders/tests/unstructured.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/unstructured.int.test.js +0 -58
- package/dist/document_loaders/tests/webpdf.int.test.d.ts +0 -1
- package/dist/document_loaders/tests/webpdf.int.test.js +0 -90
- package/dist/document_transformers/tests/openai_functions.int.test.d.ts +0 -1
- package/dist/document_transformers/tests/openai_functions.int.test.js +0 -40
- package/dist/embeddings/tests/cache.test.d.ts +0 -1
- package/dist/embeddings/tests/cache.test.js +0 -24
- package/dist/embeddings/tests/fake.test.d.ts +0 -1
- package/dist/embeddings/tests/fake.test.js +0 -34
- package/dist/evaluation/agents/tests/trajectory_eval_chain.int.test.d.ts +0 -1
- package/dist/evaluation/agents/tests/trajectory_eval_chain.int.test.js +0 -33
- package/dist/evaluation/comparison/tests/pairwise_eval_chain.int.test.d.ts +0 -1
- package/dist/evaluation/comparison/tests/pairwise_eval_chain.int.test.js +0 -46
- package/dist/evaluation/criteria/tests/criteria_eval_chain.int.test.d.ts +0 -1
- package/dist/evaluation/criteria/tests/criteria_eval_chain.int.test.js +0 -108
- package/dist/evaluation/embedding_distance/tests/embedding_distance_eval_chain.int.test.d.ts +0 -1
- package/dist/evaluation/embedding_distance/tests/embedding_distance_eval_chain.int.test.js +0 -26
- package/dist/evaluation/qa/tests/eval_chain.int.test.d.ts +0 -1
- package/dist/evaluation/qa/tests/eval_chain.int.test.js +0 -27
- package/dist/experimental/autogpt/tests/output_parser.test.d.ts +0 -1
- package/dist/experimental/autogpt/tests/output_parser.test.js +0 -8
- package/dist/experimental/autogpt/tests/prompt.test.d.ts +0 -1
- package/dist/experimental/autogpt/tests/prompt.test.js +0 -69
- package/dist/experimental/autogpt/tests/prompt_generator.test.d.ts +0 -1
- package/dist/experimental/autogpt/tests/prompt_generator.test.js +0 -91
- package/dist/experimental/chains/tests/violation_of_expectations_chain.int.test.d.ts +0 -1
- package/dist/experimental/chains/tests/violation_of_expectations_chain.int.test.js +0 -26
- package/dist/experimental/generative_agents/tests/generative_agent.int.test.d.ts +0 -1
- package/dist/experimental/generative_agents/tests/generative_agent.int.test.js +0 -304
- package/dist/experimental/masking/tests/masking-extended.test.d.ts +0 -1
- package/dist/experimental/masking/tests/masking-extended.test.js +0 -58
- package/dist/experimental/masking/tests/masking.test.d.ts +0 -1
- package/dist/experimental/masking/tests/masking.test.js +0 -388
- package/dist/experimental/openai_assistant/tests/openai_assistant.int.test.d.ts +0 -1
- package/dist/experimental/openai_assistant/tests/openai_assistant.int.test.js +0 -203
- package/dist/experimental/openai_files/tests/openai_file.int.test.d.ts +0 -1
- package/dist/experimental/openai_files/tests/openai_file.int.test.js +0 -87
- package/dist/experimental/plan_and_execute/tests/plan_and_execute.int.test.d.ts +0 -1
- package/dist/experimental/plan_and_execute/tests/plan_and_execute.int.test.js +0 -54
- package/dist/experimental/prompts/tests/handlebars.test.d.ts +0 -1
- package/dist/experimental/prompts/tests/handlebars.test.js +0 -24
- package/dist/experimental/tools/tests/pyinterpreter.int.test.d.ts +0 -1
- package/dist/experimental/tools/tests/pyinterpreter.int.test.js +0 -22
- package/dist/load/tests/cross_language.test.d.ts +0 -1
- package/dist/load/tests/cross_language.test.js +0 -83
- package/dist/load/tests/load.int.test.d.ts +0 -1
- package/dist/load/tests/load.int.test.js +0 -9
- package/dist/load/tests/load.test.d.ts +0 -1
- package/dist/load/tests/load.test.js +0 -412
- package/dist/memory/tests/buffer_memory.test.d.ts +0 -1
- package/dist/memory/tests/buffer_memory.test.js +0 -34
- package/dist/memory/tests/buffer_token_memory.int.test.d.ts +0 -1
- package/dist/memory/tests/buffer_token_memory.int.test.js +0 -47
- package/dist/memory/tests/buffer_window_memory.test.d.ts +0 -1
- package/dist/memory/tests/buffer_window_memory.test.js +0 -42
- package/dist/memory/tests/combined_memory.int.test.d.ts +0 -1
- package/dist/memory/tests/combined_memory.int.test.js +0 -74
- package/dist/memory/tests/entity_memory.int.test.d.ts +0 -1
- package/dist/memory/tests/entity_memory.int.test.js +0 -79
- package/dist/memory/tests/entity_memory.test.d.ts +0 -1
- package/dist/memory/tests/entity_memory.test.js +0 -48
- package/dist/memory/tests/summary.int.test.d.ts +0 -1
- package/dist/memory/tests/summary.int.test.js +0 -50
- package/dist/memory/tests/summary_buffer.int.test.d.ts +0 -1
- package/dist/memory/tests/summary_buffer.int.test.js +0 -55
- package/dist/memory/tests/vector_store_memory.int.test.d.ts +0 -1
- package/dist/memory/tests/vector_store_memory.int.test.js +0 -55
- package/dist/output_parsers/tests/combining.int.test.d.ts +0 -1
- package/dist/output_parsers/tests/combining.int.test.js +0 -26
- package/dist/output_parsers/tests/combining.test.d.ts +0 -1
- package/dist/output_parsers/tests/combining.test.js +0 -54
- package/dist/output_parsers/tests/datetime.test.d.ts +0 -1
- package/dist/output_parsers/tests/datetime.test.js +0 -14
- package/dist/output_parsers/tests/expression.test.d.ts +0 -1
- package/dist/output_parsers/tests/expression.test.js +0 -339
- package/dist/output_parsers/tests/http_response.test.d.ts +0 -1
- package/dist/output_parsers/tests/http_response.test.js +0 -39
- package/dist/output_parsers/tests/list.test.d.ts +0 -1
- package/dist/output_parsers/tests/list.test.js +0 -27
- package/dist/output_parsers/tests/openai_functions.int.test.d.ts +0 -1
- package/dist/output_parsers/tests/openai_functions.int.test.js +0 -89
- package/dist/output_parsers/tests/openai_tools.int.test.d.ts +0 -1
- package/dist/output_parsers/tests/openai_tools.int.test.js +0 -36
- package/dist/output_parsers/tests/structured.int.test.d.ts +0 -1
- package/dist/output_parsers/tests/structured.int.test.js +0 -150
- package/dist/prompts/tests/selectors.test.d.ts +0 -1
- package/dist/prompts/tests/selectors.test.js +0 -59
- package/dist/retrievers/self_query/tests/memory_self_query.int.test.d.ts +0 -1
- package/dist/retrievers/self_query/tests/memory_self_query.int.test.js +0 -330
- package/dist/retrievers/tests/chain_extract.int.test.d.ts +0 -1
- package/dist/retrievers/tests/chain_extract.int.test.js +0 -32
- package/dist/retrievers/tests/ensemble_retriever.int.test.d.ts +0 -1
- package/dist/retrievers/tests/ensemble_retriever.int.test.js +0 -74
- package/dist/retrievers/tests/hyde.int.test.d.ts +0 -1
- package/dist/retrievers/tests/hyde.int.test.js +0 -44
- package/dist/retrievers/tests/matryoshka_retriever.int.test.d.ts +0 -1
- package/dist/retrievers/tests/matryoshka_retriever.int.test.js +0 -113
- package/dist/retrievers/tests/multi_query.int.test.d.ts +0 -1
- package/dist/retrievers/tests/multi_query.int.test.js +0 -45
- package/dist/retrievers/tests/parent_document.int.test.d.ts +0 -1
- package/dist/retrievers/tests/parent_document.int.test.js +0 -122
- package/dist/retrievers/tests/score_threshold.int.test.d.ts +0 -1
- package/dist/retrievers/tests/score_threshold.int.test.js +0 -83
- package/dist/retrievers/tests/time_weighted.test.d.ts +0 -1
- package/dist/retrievers/tests/time_weighted.test.js +0 -320
- package/dist/retrievers/tests/vectorstores.test.d.ts +0 -1
- package/dist/retrievers/tests/vectorstores.test.js +0 -50
- package/dist/smith/tests/run_on_dataset.int.test.d.ts +0 -1
- package/dist/smith/tests/run_on_dataset.int.test.js +0 -257
- package/dist/smith/tests/runner_utils.int.test.d.ts +0 -9
- package/dist/smith/tests/runner_utils.int.test.js +0 -234
- package/dist/storage/tests/file_system.test.d.ts +0 -1
- package/dist/storage/tests/file_system.test.js +0 -81
- package/dist/tools/tests/chain.test.d.ts +0 -1
- package/dist/tools/tests/chain.test.js +0 -136
- package/dist/tools/tests/webbrowser.int.test.d.ts +0 -1
- package/dist/tools/tests/webbrowser.int.test.js +0 -80
- package/dist/tools/tests/webbrowser.test.d.ts +0 -1
- package/dist/tools/tests/webbrowser.test.js +0 -21
- package/dist/util/tests/async_caller.int.test.d.ts +0 -1
- package/dist/util/tests/async_caller.int.test.js +0 -34
- package/dist/util/tests/azure.test.d.ts +0 -1
- package/dist/util/tests/azure.test.js +0 -42
- package/dist/util/tests/openai-stream.test.d.ts +0 -1
- package/dist/util/tests/openai-stream.test.js +0 -135
- package/dist/util/tests/set.test.d.ts +0 -1
- package/dist/util/tests/set.test.js +0 -36
- package/dist/util/tests/sql_utils.test.d.ts +0 -1
- package/dist/util/tests/sql_utils.test.js +0 -50
- package/dist/vectorstores/tests/memory.test.d.ts +0 -1
- package/dist/vectorstores/tests/memory.test.js +0 -78
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
import * as url from "node:url";
|
|
2
|
-
import * as path from "node:path";
|
|
3
|
-
import { test, expect } from "@jest/globals";
|
|
4
|
-
import { DirectoryLoader, UnknownHandling } from "../fs/directory.js";
|
|
5
|
-
import { CSVLoader } from "../fs/csv.js";
|
|
6
|
-
import { PDFLoader } from "../fs/pdf.js";
|
|
7
|
-
import { TextLoader } from "../fs/text.js";
|
|
8
|
-
import { JSONLoader } from "../fs/json.js";
|
|
9
|
-
test("Test Directory loader", async () => {
|
|
10
|
-
const directoryPath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data");
|
|
11
|
-
const loader = new DirectoryLoader(directoryPath, {
|
|
12
|
-
".csv": (p) => {
|
|
13
|
-
if (p.includes("separator.csv")) {
|
|
14
|
-
return new CSVLoader(p, { column: "html", separator: "|" });
|
|
15
|
-
}
|
|
16
|
-
return new CSVLoader(p, "html");
|
|
17
|
-
},
|
|
18
|
-
".pdf": (p) => new PDFLoader(p),
|
|
19
|
-
".txt": (p) => new TextLoader(p),
|
|
20
|
-
".json": (p) => new JSONLoader(p),
|
|
21
|
-
}, false, UnknownHandling.Ignore);
|
|
22
|
-
const docs = await loader.load();
|
|
23
|
-
expect(docs.length).toBe(123);
|
|
24
|
-
expect(docs.map((d) => d.metadata.source).sort()).toEqual([
|
|
25
|
-
// PDF
|
|
26
|
-
...Array.from({ length: 15 }, (_) => path.resolve(directoryPath, "1706.03762.pdf")),
|
|
27
|
-
path.resolve(directoryPath, "Jacob_Lee_Resume_2023.pdf"),
|
|
28
|
-
// CSV
|
|
29
|
-
...Array.from({ length: 32 }, (_) => path.resolve(directoryPath, "Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.csv")),
|
|
30
|
-
// JSON
|
|
31
|
-
...Array.from({ length: 32 }, (_) => path.resolve(directoryPath, "Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.json")),
|
|
32
|
-
...Array.from({ length: 10 }, (_) => path.resolve(directoryPath, "complex.json")),
|
|
33
|
-
// TXT
|
|
34
|
-
path.resolve(directoryPath, "example.txt"),
|
|
35
|
-
// CSV
|
|
36
|
-
...Array.from({ length: 32 }, (_) => path.resolve(directoryPath, "example_separator.csv")),
|
|
37
|
-
]);
|
|
38
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { test, expect } from "@jest/globals";
|
|
2
|
-
import * as url from "node:url";
|
|
3
|
-
import * as path from "node:path";
|
|
4
|
-
import { DocxLoader } from "../fs/docx.js";
|
|
5
|
-
test("Test Word doc loader from file", async () => {
|
|
6
|
-
const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/attention.docx");
|
|
7
|
-
const loader = new DocxLoader(filePath);
|
|
8
|
-
const docs = await loader.load();
|
|
9
|
-
expect(docs.length).toBe(1); // not much text in the example
|
|
10
|
-
expect(docs[0].pageContent).toContain("an interesting activity");
|
|
11
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import { test, expect } from "@jest/globals";
|
|
2
|
-
import * as url from "node:url";
|
|
3
|
-
import * as path from "node:path";
|
|
4
|
-
import { EPubLoader } from "../fs/epub.js";
|
|
5
|
-
test("Test EPub loader from file", async () => {
|
|
6
|
-
const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/attention.epub");
|
|
7
|
-
const loader = new EPubLoader(filePath);
|
|
8
|
-
const docs = await loader.load();
|
|
9
|
-
expect(docs.length).toBe(3);
|
|
10
|
-
expect(docs[0].pageContent).toContain("Attention Is All You Need");
|
|
11
|
-
});
|
|
12
|
-
test("Test EPub loader from file to single document", async () => {
|
|
13
|
-
const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/attention.epub");
|
|
14
|
-
const loader = new EPubLoader(filePath, { splitChapters: false });
|
|
15
|
-
const docs = await loader.load();
|
|
16
|
-
expect(docs.length).toBe(1);
|
|
17
|
-
expect(docs[0].pageContent).toContain("Attention Is All You Need");
|
|
18
|
-
});
|
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
export const GithubLoaderApis = {
|
|
2
|
-
getRepoFiles: {
|
|
3
|
-
0: [
|
|
4
|
-
{
|
|
5
|
-
name: "foo.txt",
|
|
6
|
-
path: "foo.txt",
|
|
7
|
-
type: "file",
|
|
8
|
-
size: 50,
|
|
9
|
-
url: "https://githubfilecontent.com",
|
|
10
|
-
html_url: "",
|
|
11
|
-
sha: "",
|
|
12
|
-
git_url: "",
|
|
13
|
-
download_url: "",
|
|
14
|
-
_links: {
|
|
15
|
-
self: "",
|
|
16
|
-
git: "",
|
|
17
|
-
html: "",
|
|
18
|
-
},
|
|
19
|
-
},
|
|
20
|
-
{
|
|
21
|
-
name: "dir1",
|
|
22
|
-
path: "dir1",
|
|
23
|
-
type: "dir",
|
|
24
|
-
size: 50,
|
|
25
|
-
url: "https://githubfilecontent.com",
|
|
26
|
-
html_url: "",
|
|
27
|
-
sha: "",
|
|
28
|
-
git_url: "",
|
|
29
|
-
download_url: "",
|
|
30
|
-
_links: {
|
|
31
|
-
self: "",
|
|
32
|
-
git: "",
|
|
33
|
-
html: "",
|
|
34
|
-
},
|
|
35
|
-
},
|
|
36
|
-
],
|
|
37
|
-
1: [
|
|
38
|
-
{
|
|
39
|
-
name: "dir1_1",
|
|
40
|
-
path: "dir1/dir1_1",
|
|
41
|
-
type: "dir",
|
|
42
|
-
size: 50,
|
|
43
|
-
url: "https://githubfilecontent.com",
|
|
44
|
-
html_url: "",
|
|
45
|
-
sha: "",
|
|
46
|
-
git_url: "",
|
|
47
|
-
download_url: "",
|
|
48
|
-
_links: {
|
|
49
|
-
self: "",
|
|
50
|
-
git: "",
|
|
51
|
-
html: "",
|
|
52
|
-
},
|
|
53
|
-
},
|
|
54
|
-
],
|
|
55
|
-
2: [
|
|
56
|
-
{
|
|
57
|
-
name: "nested_file.txt",
|
|
58
|
-
path: "dir1/dir1_1/nested_file.txt",
|
|
59
|
-
type: "file",
|
|
60
|
-
size: 50,
|
|
61
|
-
url: "https://githubfilecontent.com",
|
|
62
|
-
html_url: "",
|
|
63
|
-
sha: "",
|
|
64
|
-
git_url: "",
|
|
65
|
-
download_url: "",
|
|
66
|
-
_links: {
|
|
67
|
-
self: "",
|
|
68
|
-
git: "",
|
|
69
|
-
html: "",
|
|
70
|
-
},
|
|
71
|
-
},
|
|
72
|
-
{
|
|
73
|
-
name: "EXAMPLE.md",
|
|
74
|
-
path: "dir1/dir1_1/EXAMPLE.md",
|
|
75
|
-
type: "file",
|
|
76
|
-
size: 50,
|
|
77
|
-
url: "https://githubfilecontent.com",
|
|
78
|
-
html_url: "",
|
|
79
|
-
sha: "",
|
|
80
|
-
git_url: "",
|
|
81
|
-
download_url: "",
|
|
82
|
-
_links: {
|
|
83
|
-
self: "",
|
|
84
|
-
git: "",
|
|
85
|
-
html: "",
|
|
86
|
-
},
|
|
87
|
-
},
|
|
88
|
-
],
|
|
89
|
-
},
|
|
90
|
-
getFileContents: "this is a file full of stuff",
|
|
91
|
-
};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
/* eslint-disable no-process-env */
|
|
2
|
-
/* eslint-disable @typescript-eslint/no-non-null-assertion */
|
|
3
|
-
import { test } from "@jest/globals";
|
|
4
|
-
import { FigmaFileLoader } from "../web/figma.js";
|
|
5
|
-
test.skip("Test FigmaFileLoader", async () => {
|
|
6
|
-
const loader = new FigmaFileLoader({
|
|
7
|
-
accessToken: process.env.FIGMA_ACCESS_TOKEN,
|
|
8
|
-
nodeIds: (process.env.FIGMA_NODE_IDS ?? "").split(","),
|
|
9
|
-
fileKey: process.env.FIGMA_FILE_KEY,
|
|
10
|
-
});
|
|
11
|
-
const documents = await loader.load();
|
|
12
|
-
console.log(documents[0].pageContent);
|
|
13
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
/* eslint-disable no-process-env */
|
|
2
|
-
/* eslint-disable @typescript-eslint/no-non-null-assertion */
|
|
3
|
-
import { test } from "@jest/globals";
|
|
4
|
-
import { Document } from "@langchain/core/documents";
|
|
5
|
-
import { FireCrawlLoader } from "../web/firecrawl.js";
|
|
6
|
-
test("Test FireCrawlLoader load method with scrape mode", async () => {
|
|
7
|
-
const loader = new FireCrawlLoader({
|
|
8
|
-
url: "https://firecrawl.dev",
|
|
9
|
-
apiKey: process.env.FIRECRAWL_API_KEY,
|
|
10
|
-
mode: "scrape",
|
|
11
|
-
});
|
|
12
|
-
const documents = await loader.load();
|
|
13
|
-
expect(documents).toHaveLength(1);
|
|
14
|
-
const document = documents[0];
|
|
15
|
-
expect(document).toBeInstanceOf(Document);
|
|
16
|
-
expect(document.pageContent).toBeTruthy();
|
|
17
|
-
expect(document.metadata).toBeTruthy();
|
|
18
|
-
});
|
|
19
|
-
test("Test FireCrawlLoader load method with crawl mode", async () => {
|
|
20
|
-
const loader = new FireCrawlLoader({
|
|
21
|
-
url: "https://firecrawl.dev",
|
|
22
|
-
apiKey: process.env.FIRECRAWL_API_KEY,
|
|
23
|
-
mode: "crawl",
|
|
24
|
-
});
|
|
25
|
-
const documents = await loader.load();
|
|
26
|
-
const document = documents[0];
|
|
27
|
-
expect(document).toBeInstanceOf(Document);
|
|
28
|
-
expect(document.pageContent).toBeTruthy();
|
|
29
|
-
expect(document.metadata).toBeTruthy();
|
|
30
|
-
}, 15000);
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { test } from "@jest/globals";
|
|
2
|
-
import { GitbookLoader } from "../web/gitbook.js";
|
|
3
|
-
test("Test GitbookLoader", async () => {
|
|
4
|
-
const loader = new GitbookLoader("https://docs.gitbook.com/product-tour/navigation");
|
|
5
|
-
const docs = await loader.load();
|
|
6
|
-
console.log("Loaded", docs.length, "Gitbook documents");
|
|
7
|
-
});
|
|
8
|
-
test("Test GitbookLoader with shouldLoadAllPaths", async () => {
|
|
9
|
-
const loader = new GitbookLoader("https://docs.maildrop.cc", {
|
|
10
|
-
shouldLoadAllPaths: true,
|
|
11
|
-
});
|
|
12
|
-
const docs = await loader.load();
|
|
13
|
-
console.log("Loaded", docs.length, "Gitbook documents");
|
|
14
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,86 +0,0 @@
|
|
|
1
|
-
import { test } from "@jest/globals";
|
|
2
|
-
import { GithubRepoLoader } from "../web/github.js";
|
|
3
|
-
test("Test GithubRepoLoader", async () => {
|
|
4
|
-
const loader = new GithubRepoLoader("https://github.com/langchain-ai/langchainjs", { branch: "main", recursive: false, unknown: "warn" });
|
|
5
|
-
const documents = await loader.load();
|
|
6
|
-
expect(documents.filter((document) => document.metadata.source === "yarn.lock")
|
|
7
|
-
.length).toBe(1);
|
|
8
|
-
expect(documents.filter((document) => document.metadata.source === "README.md")
|
|
9
|
-
.length).toBe(1);
|
|
10
|
-
console.log(documents[0].pageContent);
|
|
11
|
-
});
|
|
12
|
-
test("Test ignoreFiles with GithubRepoLoader", async () => {
|
|
13
|
-
const loader = new GithubRepoLoader("https://github.com/langchain-ai/langchainjs", {
|
|
14
|
-
branch: "main",
|
|
15
|
-
recursive: false,
|
|
16
|
-
unknown: "warn",
|
|
17
|
-
ignoreFiles: ["yarn.lock", "README.md"],
|
|
18
|
-
});
|
|
19
|
-
const documents = await loader.load();
|
|
20
|
-
expect(documents.filter((document) => document.metadata.source === "yarn.lock")
|
|
21
|
-
.length).toBe(0);
|
|
22
|
-
expect(documents.filter((document) => document.metadata.source === "README.md")
|
|
23
|
-
.length).toBe(0);
|
|
24
|
-
console.log(documents[0].pageContent);
|
|
25
|
-
});
|
|
26
|
-
test("Test ignorePaths with GithubRepoLoader", async () => {
|
|
27
|
-
const loader = new GithubRepoLoader("https://github.com/langchain-ai/langchainjs", {
|
|
28
|
-
branch: "main",
|
|
29
|
-
recursive: false,
|
|
30
|
-
unknown: "warn",
|
|
31
|
-
ignorePaths: ["yarn.lock", "*.md"],
|
|
32
|
-
});
|
|
33
|
-
const documents = await loader.load();
|
|
34
|
-
expect(documents.filter((document) => document.metadata.source === "yarn.lock")
|
|
35
|
-
.length).toBe(0);
|
|
36
|
-
expect(documents.filter((document) => document.metadata.source.endsWith(".md"))
|
|
37
|
-
.length).toBe(0);
|
|
38
|
-
console.log(documents[0].pageContent);
|
|
39
|
-
});
|
|
40
|
-
test("Test streaming documents from GithubRepoLoader", async () => {
|
|
41
|
-
const loader = new GithubRepoLoader("https://github.com/langchain-ai/langchainjs", {
|
|
42
|
-
branch: "main",
|
|
43
|
-
recursive: false,
|
|
44
|
-
unknown: "warn",
|
|
45
|
-
});
|
|
46
|
-
const documents = [];
|
|
47
|
-
for await (const document of loader.loadAsStream()) {
|
|
48
|
-
documents.push(document);
|
|
49
|
-
}
|
|
50
|
-
expect(documents.filter((document) => document.metadata.source === "yarn.lock")
|
|
51
|
-
.length).toBe(1);
|
|
52
|
-
expect(documents.filter((document) => document.metadata.source === "README.md")
|
|
53
|
-
.length).toBe(1);
|
|
54
|
-
});
|
|
55
|
-
test("Test ignorePaths streaming with GithubRepoLoader", async () => {
|
|
56
|
-
const loader = new GithubRepoLoader("https://github.com/langchain-ai/langchainjs", {
|
|
57
|
-
branch: "main",
|
|
58
|
-
recursive: false,
|
|
59
|
-
unknown: "warn",
|
|
60
|
-
ignorePaths: ["yarn.lock", "*.md"],
|
|
61
|
-
});
|
|
62
|
-
const documents = [];
|
|
63
|
-
for await (const document of loader.loadAsStream()) {
|
|
64
|
-
documents.push(document);
|
|
65
|
-
}
|
|
66
|
-
expect(documents.filter((document) => document.metadata.source === "yarn.lock")
|
|
67
|
-
.length).toBe(0);
|
|
68
|
-
expect(documents.filter((document) => document.metadata.source.endsWith(".md"))
|
|
69
|
-
.length).toBe(0);
|
|
70
|
-
});
|
|
71
|
-
test("Test ignoreFiles streaming with GithubRepoLoader", async () => {
|
|
72
|
-
const loader = new GithubRepoLoader("https://github.com/langchain-ai/langchainjs", {
|
|
73
|
-
branch: "main",
|
|
74
|
-
recursive: false,
|
|
75
|
-
unknown: "warn",
|
|
76
|
-
ignoreFiles: ["yarn.lock", "README.md"],
|
|
77
|
-
});
|
|
78
|
-
const documents = [];
|
|
79
|
-
for await (const document of loader.loadAsStream()) {
|
|
80
|
-
documents.push(document);
|
|
81
|
-
}
|
|
82
|
-
expect(documents.filter((document) => document.metadata.source === "yarn.lock")
|
|
83
|
-
.length).toBe(0);
|
|
84
|
-
expect(documents.filter((document) => document.metadata.source === "README.md")
|
|
85
|
-
.length).toBe(0);
|
|
86
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
import { jest, test } from "@jest/globals";
|
|
2
|
-
import { GithubRepoLoader } from "../web/github.js";
|
|
3
|
-
import { GithubLoaderApis } from "./example_data/github_api_responses.js";
|
|
4
|
-
describe("GithubRepoLoader recursion", () => {
|
|
5
|
-
let callCount = 0;
|
|
6
|
-
beforeAll(() => {
|
|
7
|
-
global.fetch = jest.fn().mockImplementation((url) => {
|
|
8
|
-
let responseData = GithubLoaderApis.getRepoFiles[callCount.toString()];
|
|
9
|
-
if (url.includes("https://api.github.com/repos")) {
|
|
10
|
-
responseData = GithubLoaderApis.getRepoFiles[callCount.toString()];
|
|
11
|
-
callCount += 1;
|
|
12
|
-
}
|
|
13
|
-
else if (url.includes("githubfilecontent.com")) {
|
|
14
|
-
responseData = GithubLoaderApis.getFileContents;
|
|
15
|
-
}
|
|
16
|
-
return Promise.resolve({
|
|
17
|
-
ok: true,
|
|
18
|
-
json: () => Promise.resolve(responseData),
|
|
19
|
-
text: () => Promise.resolve("this is a file full of stuff"),
|
|
20
|
-
});
|
|
21
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
22
|
-
});
|
|
23
|
-
});
|
|
24
|
-
afterAll(() => {
|
|
25
|
-
jest.clearAllMocks();
|
|
26
|
-
callCount = 0;
|
|
27
|
-
});
|
|
28
|
-
test("Test recursion with GithubRepoLoader", async () => {
|
|
29
|
-
const loader = new GithubRepoLoader("https://github.com/langchain-ai/langchainjs", {
|
|
30
|
-
branch: "main",
|
|
31
|
-
recursive: true,
|
|
32
|
-
unknown: "warn",
|
|
33
|
-
ignorePaths: ["*.md"],
|
|
34
|
-
});
|
|
35
|
-
const documents = await loader.load();
|
|
36
|
-
expect(documents.length).toBe(2);
|
|
37
|
-
expect(documents.map((doc) => doc.metadata.source)).toEqual([
|
|
38
|
-
"foo.txt",
|
|
39
|
-
"dir1/dir1_1/nested_file.txt",
|
|
40
|
-
]);
|
|
41
|
-
});
|
|
42
|
-
test("Expect an error if processSubmodules set without recursive with GithubRepoLoader", async () => {
|
|
43
|
-
expect(() => new GithubRepoLoader("https://github.com/langchain-ai/langchainjs", {
|
|
44
|
-
branch: "main",
|
|
45
|
-
recursive: false,
|
|
46
|
-
processSubmodules: true,
|
|
47
|
-
unknown: "warn",
|
|
48
|
-
ignorePaths: ["*.md"],
|
|
49
|
-
})).toThrow();
|
|
50
|
-
});
|
|
51
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
import * as url from "node:url";
|
|
2
|
-
import * as path from "node:path";
|
|
3
|
-
import * as fs from "node:fs/promises";
|
|
4
|
-
import { test, expect } from "@jest/globals";
|
|
5
|
-
import { Document } from "@langchain/core/documents";
|
|
6
|
-
import { JSONLoader } from "../fs/json.js";
|
|
7
|
-
test("Test JSON loader from blob", async () => {
|
|
8
|
-
const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.json");
|
|
9
|
-
const loader = new JSONLoader(new Blob([await fs.readFile(filePath)], { type: "application/json" }));
|
|
10
|
-
const docs = await loader.load();
|
|
11
|
-
expect(docs.length).toBe(32);
|
|
12
|
-
expect(docs[0]).toEqual(new Document({
|
|
13
|
-
metadata: { source: "blob", blobType: "application/json", line: 1 },
|
|
14
|
-
pageContent: "<i>Corruption discovered at the core of the Banking Clan!</i>",
|
|
15
|
-
}));
|
|
16
|
-
});
|
|
17
|
-
test("Test JSON loader from blob", async () => {
|
|
18
|
-
const loader = new JSONLoader(new Blob([
|
|
19
|
-
`{
|
|
20
|
-
"texts": ["This is a sentence.", "This is another sentence."]
|
|
21
|
-
}`,
|
|
22
|
-
], { type: "application/json" }));
|
|
23
|
-
const docs = await loader.load();
|
|
24
|
-
expect(docs.length).toBe(2);
|
|
25
|
-
expect(docs[0]).toMatchInlineSnapshot(`
|
|
26
|
-
Document {
|
|
27
|
-
"id": undefined,
|
|
28
|
-
"metadata": {
|
|
29
|
-
"blobType": "application/json",
|
|
30
|
-
"line": 1,
|
|
31
|
-
"source": "blob",
|
|
32
|
-
},
|
|
33
|
-
"pageContent": "This is a sentence.",
|
|
34
|
-
}
|
|
35
|
-
`);
|
|
36
|
-
expect(docs[1]).toMatchInlineSnapshot(`
|
|
37
|
-
Document {
|
|
38
|
-
"id": undefined,
|
|
39
|
-
"metadata": {
|
|
40
|
-
"blobType": "application/json",
|
|
41
|
-
"line": 2,
|
|
42
|
-
"source": "blob",
|
|
43
|
-
},
|
|
44
|
-
"pageContent": "This is another sentence.",
|
|
45
|
-
}
|
|
46
|
-
`);
|
|
47
|
-
});
|
|
48
|
-
test("Test JSON loader from blob", async () => {
|
|
49
|
-
const loader = new JSONLoader(new Blob([
|
|
50
|
-
`{
|
|
51
|
-
"1": {
|
|
52
|
-
"body": "BD 2023 SUMMER",
|
|
53
|
-
"from": "LinkedIn Job",
|
|
54
|
-
"labels": ["IMPORTANT", "CATEGORY_UPDATES", "INBOX"]
|
|
55
|
-
},
|
|
56
|
-
"2": {
|
|
57
|
-
"body": "Intern, Treasury and other roles are available",
|
|
58
|
-
"from": "LinkedIn Job2",
|
|
59
|
-
"labels": ["IMPORTANT"],
|
|
60
|
-
"other": {
|
|
61
|
-
"name": "plop",
|
|
62
|
-
"surname": "bob"
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
}`,
|
|
66
|
-
], { type: "application/json" }));
|
|
67
|
-
const docs = await loader.load();
|
|
68
|
-
expect(docs.length).toBe(10);
|
|
69
|
-
expect(docs[0]).toMatchInlineSnapshot(`
|
|
70
|
-
Document {
|
|
71
|
-
"id": undefined,
|
|
72
|
-
"metadata": {
|
|
73
|
-
"blobType": "application/json",
|
|
74
|
-
"line": 1,
|
|
75
|
-
"source": "blob",
|
|
76
|
-
},
|
|
77
|
-
"pageContent": "BD 2023 SUMMER",
|
|
78
|
-
}
|
|
79
|
-
`);
|
|
80
|
-
expect(docs[1]).toMatchInlineSnapshot(`
|
|
81
|
-
Document {
|
|
82
|
-
"id": undefined,
|
|
83
|
-
"metadata": {
|
|
84
|
-
"blobType": "application/json",
|
|
85
|
-
"line": 2,
|
|
86
|
-
"source": "blob",
|
|
87
|
-
},
|
|
88
|
-
"pageContent": "LinkedIn Job",
|
|
89
|
-
}
|
|
90
|
-
`);
|
|
91
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,69 +0,0 @@
|
|
|
1
|
-
import * as url from "node:url";
|
|
2
|
-
import * as path from "node:path";
|
|
3
|
-
import { test, expect } from "@jest/globals";
|
|
4
|
-
import { Document } from "@langchain/core/documents";
|
|
5
|
-
import { JSONLoader } from "../fs/json.js";
|
|
6
|
-
test("Test JSON loader", async () => {
|
|
7
|
-
const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.json");
|
|
8
|
-
const loader = new JSONLoader(filePath);
|
|
9
|
-
const docs = await loader.load();
|
|
10
|
-
expect(docs.length).toBe(32);
|
|
11
|
-
expect(docs[0]).toEqual(new Document({
|
|
12
|
-
metadata: { source: filePath, line: 1 },
|
|
13
|
-
pageContent: "<i>Corruption discovered at the core of the Banking Clan!</i>",
|
|
14
|
-
}));
|
|
15
|
-
});
|
|
16
|
-
test("Test JSON loader for complex json without keys", async () => {
|
|
17
|
-
const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/complex.json");
|
|
18
|
-
const loader = new JSONLoader(filePath);
|
|
19
|
-
const docs = await loader.load();
|
|
20
|
-
expect(docs.length).toBe(10);
|
|
21
|
-
expect(docs[0]).toEqual(new Document({
|
|
22
|
-
metadata: { source: filePath, line: 1 },
|
|
23
|
-
pageContent: "BD 2023 SUMMER",
|
|
24
|
-
}));
|
|
25
|
-
expect(docs[1]).toEqual(new Document({
|
|
26
|
-
metadata: { source: filePath, line: 2 },
|
|
27
|
-
pageContent: "LinkedIn Job",
|
|
28
|
-
}));
|
|
29
|
-
expect(docs[2]).toEqual(new Document({
|
|
30
|
-
metadata: { source: filePath, line: 3 },
|
|
31
|
-
pageContent: "IMPORTANT",
|
|
32
|
-
}));
|
|
33
|
-
});
|
|
34
|
-
test("Test JSON loader for complex json with one key that points nothing", async () => {
|
|
35
|
-
const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/complex.json");
|
|
36
|
-
const loader = new JSONLoader(filePath, ["/plop"]);
|
|
37
|
-
const docs = await loader.load();
|
|
38
|
-
expect(docs.length).toBe(0);
|
|
39
|
-
});
|
|
40
|
-
test("Test JSON loader for complex json with one key that exists", async () => {
|
|
41
|
-
const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/complex.json");
|
|
42
|
-
const loader = new JSONLoader(filePath, ["/from"]);
|
|
43
|
-
const docs = await loader.load();
|
|
44
|
-
expect(docs.length).toBe(2);
|
|
45
|
-
expect(docs[1]).toEqual(new Document({
|
|
46
|
-
metadata: { source: filePath, line: 2 },
|
|
47
|
-
pageContent: "LinkedIn Job2",
|
|
48
|
-
}));
|
|
49
|
-
});
|
|
50
|
-
test("Test JSON loader for complex json with two keys that exists", async () => {
|
|
51
|
-
const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/complex.json");
|
|
52
|
-
const loader = new JSONLoader(filePath, ["/from", "/labels"]);
|
|
53
|
-
const docs = await loader.load();
|
|
54
|
-
expect(docs.length).toBe(6);
|
|
55
|
-
expect(docs[3]).toEqual(new Document({
|
|
56
|
-
metadata: { source: filePath, line: 4 },
|
|
57
|
-
pageContent: "INBOX",
|
|
58
|
-
}));
|
|
59
|
-
});
|
|
60
|
-
test("Test JSON loader for complex json with two existing keys on different level", async () => {
|
|
61
|
-
const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/complex.json");
|
|
62
|
-
const loader = new JSONLoader(filePath, ["/from", "/surname"]);
|
|
63
|
-
const docs = await loader.load();
|
|
64
|
-
expect(docs.length).toBe(3);
|
|
65
|
-
expect(docs[2]).toEqual(new Document({
|
|
66
|
-
metadata: { source: filePath, line: 3 },
|
|
67
|
-
pageContent: "bob",
|
|
68
|
-
}));
|
|
69
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
import * as url from "node:url";
|
|
2
|
-
import * as path from "node:path";
|
|
3
|
-
import * as fs from "node:fs/promises";
|
|
4
|
-
import { test, expect } from "@jest/globals";
|
|
5
|
-
import { Document } from "@langchain/core/documents";
|
|
6
|
-
import { JSONLinesLoader } from "../fs/json.js";
|
|
7
|
-
test("Test JSONL loader from blob", async () => {
|
|
8
|
-
const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.jsonl");
|
|
9
|
-
const loader = new JSONLinesLoader(new Blob([await fs.readFile(filePath)], { type: "application/jsonl+json" }), "/html");
|
|
10
|
-
const docs = await loader.load();
|
|
11
|
-
expect(docs.length).toBe(32);
|
|
12
|
-
expect(docs[0]).toEqual(new Document({
|
|
13
|
-
metadata: { source: "blob", blobType: "application/jsonl+json", line: 1 },
|
|
14
|
-
pageContent: "<i>Corruption discovered at the core of the Banking Clan!</i>",
|
|
15
|
-
}));
|
|
16
|
-
});
|
|
17
|
-
test("Test JSONL loader from blob", async () => {
|
|
18
|
-
const loader = new JSONLinesLoader(new Blob([
|
|
19
|
-
`{"html": "This is a sentence."}
|
|
20
|
-
{"html": "This is another sentence."}`,
|
|
21
|
-
], { type: "application/jsonl+json" }), "/html");
|
|
22
|
-
const docs = await loader.load();
|
|
23
|
-
expect(docs.length).toBe(2);
|
|
24
|
-
expect(docs[0]).toMatchInlineSnapshot(`
|
|
25
|
-
Document {
|
|
26
|
-
"id": undefined,
|
|
27
|
-
"metadata": {
|
|
28
|
-
"blobType": "application/jsonl+json",
|
|
29
|
-
"line": 1,
|
|
30
|
-
"source": "blob",
|
|
31
|
-
},
|
|
32
|
-
"pageContent": "This is a sentence.",
|
|
33
|
-
}
|
|
34
|
-
`);
|
|
35
|
-
expect(docs[1]).toMatchInlineSnapshot(`
|
|
36
|
-
Document {
|
|
37
|
-
"id": undefined,
|
|
38
|
-
"metadata": {
|
|
39
|
-
"blobType": "application/jsonl+json",
|
|
40
|
-
"line": 2,
|
|
41
|
-
"source": "blob",
|
|
42
|
-
},
|
|
43
|
-
"pageContent": "This is another sentence.",
|
|
44
|
-
}
|
|
45
|
-
`);
|
|
46
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import * as url from "node:url";
|
|
2
|
-
import * as path from "node:path";
|
|
3
|
-
import { test, expect } from "@jest/globals";
|
|
4
|
-
import { Document } from "@langchain/core/documents";
|
|
5
|
-
import { JSONLinesLoader } from "../fs/json.js";
|
|
6
|
-
test("Test JSON loader from file", async () => {
|
|
7
|
-
const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.jsonl");
|
|
8
|
-
const loader = new JSONLinesLoader(filePath, "/html");
|
|
9
|
-
const docs = await loader.load();
|
|
10
|
-
expect(docs.length).toBe(32);
|
|
11
|
-
expect(docs[0]).toEqual(new Document({
|
|
12
|
-
metadata: { source: filePath, line: 1 },
|
|
13
|
-
pageContent: "<i>Corruption discovered at the core of the Banking Clan!</i>",
|
|
14
|
-
}));
|
|
15
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|