@deepagents/retrieval 7.5.2 → 7.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +24 -17
- package/dist/index.js.map +2 -2
- package/dist/lib/connectors/connector.d.ts +22 -8
- package/dist/lib/connectors/connector.d.ts.map +1 -1
- package/dist/lib/connectors/index.js +7 -9
- package/dist/lib/connectors/index.js.map +2 -2
- package/dist/lib/connectors/local.d.ts +2 -4
- package/dist/lib/connectors/local.d.ts.map +1 -1
- package/dist/lib/connectors/repo.d.ts +1 -1
- package/dist/lib/connectors/repo.d.ts.map +1 -1
- package/dist/lib/connectors/rss.d.ts.map +1 -1
- package/dist/lib/ingest.d.ts.map +1 -1
- package/dist/lib/similiarty-search.d.ts.map +1 -1
- package/dist/lib/stores/sqlite/sqlite.d.ts +1 -1
- package/dist/lib/stores/sqlite/sqlite.d.ts.map +1 -1
- package/dist/lib/stores/store.d.ts +1 -1
- package/dist/lib/stores/store.d.ts.map +1 -1
- package/package.json +16 -12
package/dist/index.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// packages/retrieval/src/lib/embedders/huggingface.ts
|
|
2
2
|
function tensorToEmbeddings(tensor) {
|
|
3
3
|
const dims = tensor.dims;
|
|
4
|
-
|
|
4
|
+
const [batchSize, ...rest] = Array.isArray(dims) ? dims : [];
|
|
5
|
+
const hiddenSize = rest.at(-1);
|
|
6
|
+
if (hiddenSize === void 0) {
|
|
5
7
|
throw new Error(`Unexpected tensor dims: ${JSON.stringify(dims)}`);
|
|
6
8
|
}
|
|
7
|
-
const batchSize = dims[0];
|
|
8
|
-
const hiddenSize = dims[dims.length - 1];
|
|
9
9
|
const expectedLen = batchSize * hiddenSize;
|
|
10
10
|
if (tensor.data.length !== expectedLen) {
|
|
11
11
|
throw new Error(
|
|
@@ -17,7 +17,7 @@ function tensorToEmbeddings(tensor) {
|
|
|
17
17
|
throw new Error(`Expected float32 embeddings, got ${tensor.type}`);
|
|
18
18
|
}
|
|
19
19
|
const embeddings = [];
|
|
20
|
-
const dimensions =
|
|
20
|
+
const dimensions = hiddenSize;
|
|
21
21
|
for (let i = 0; i < batchSize; i++) {
|
|
22
22
|
const start = i * hiddenSize;
|
|
23
23
|
embeddings.push(data.subarray(start, start + hiddenSize));
|
|
@@ -91,9 +91,9 @@ import {
|
|
|
91
91
|
} from "@langchain/textsplitters";
|
|
92
92
|
|
|
93
93
|
// packages/retrieval/src/lib/stores/cid.ts
|
|
94
|
-
import {
|
|
94
|
+
import { hash } from "node:crypto";
|
|
95
95
|
function cid(content) {
|
|
96
|
-
return
|
|
96
|
+
return hash("sha256", content);
|
|
97
97
|
}
|
|
98
98
|
|
|
99
99
|
// packages/retrieval/src/lib/ingest.ts
|
|
@@ -101,8 +101,7 @@ async function ingest(config, callback) {
|
|
|
101
101
|
const splitter = config.splitter ?? split;
|
|
102
102
|
const embedder = config.embedder;
|
|
103
103
|
const corpuses = config.connector.sources();
|
|
104
|
-
const
|
|
105
|
-
const expiryDate = expiresAfter === void 0 ? void 0 : new Date(Date.now() + expiresAfter);
|
|
104
|
+
const expiryDate = expiry(config.connector);
|
|
106
105
|
for await (const it of corpuses) {
|
|
107
106
|
callback?.(it.id);
|
|
108
107
|
const content = await it.content();
|
|
@@ -134,6 +133,11 @@ async function ingest(config, callback) {
|
|
|
134
133
|
);
|
|
135
134
|
}
|
|
136
135
|
}
|
|
136
|
+
function expiry(connector) {
|
|
137
|
+
if (connector.ingestWhen !== "expired") return void 0;
|
|
138
|
+
if (connector.expiresAfter === void 0) return void 0;
|
|
139
|
+
return new Date(connector.clock.Date.now() + connector.expiresAfter);
|
|
140
|
+
}
|
|
137
141
|
function split(id, content) {
|
|
138
142
|
const splitter = new MarkdownTextSplitter();
|
|
139
143
|
return splitter.splitText(content);
|
|
@@ -173,10 +177,10 @@ function computePositions(originalContent, chunks) {
|
|
|
173
177
|
const before = normalizedContent.slice(0, matchIndex);
|
|
174
178
|
const beforeLines = before.split("\n");
|
|
175
179
|
const startLine = beforeLines.length;
|
|
176
|
-
const startColumn =
|
|
180
|
+
const startColumn = before.length - before.lastIndexOf("\n");
|
|
177
181
|
const lines = matchValue.split("\n");
|
|
178
182
|
const endLine = startLine + lines.length - 1;
|
|
179
|
-
const endColumn = lines.length === 1 ? startColumn + lines[0].length :
|
|
183
|
+
const endColumn = lines.length === 1 ? startColumn + lines[0].length : matchValue.length - matchValue.lastIndexOf("\n");
|
|
180
184
|
positions.push({ startLine, startColumn, endLine, endColumn });
|
|
181
185
|
searchOffset = matchIndex + matchValue.length;
|
|
182
186
|
}
|
|
@@ -207,20 +211,23 @@ async function splitTypeScript(id, content) {
|
|
|
207
211
|
// packages/retrieval/src/lib/similiarty-search.ts
|
|
208
212
|
import "ai";
|
|
209
213
|
async function similaritySearch(query, config) {
|
|
210
|
-
const
|
|
214
|
+
const { connector } = config;
|
|
211
215
|
let shouldIngest = true;
|
|
212
|
-
if (
|
|
216
|
+
if (connector.ingestWhen === "never") {
|
|
213
217
|
if (await config.store.sourceExists(config.connector.sourceId)) {
|
|
214
218
|
console.log(
|
|
215
219
|
`Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=never and source exists)`
|
|
216
220
|
);
|
|
217
221
|
shouldIngest = false;
|
|
218
222
|
}
|
|
219
|
-
} else if (
|
|
223
|
+
} else if (connector.ingestWhen === "expired") {
|
|
220
224
|
const sourceExists = await config.store.sourceExists(
|
|
221
225
|
config.connector.sourceId
|
|
222
226
|
);
|
|
223
|
-
if (sourceExists && !await config.store.sourceExpired(
|
|
227
|
+
if (sourceExists && !await config.store.sourceExpired(
|
|
228
|
+
config.connector.sourceId,
|
|
229
|
+
new Date(connector.clock.Date.now())
|
|
230
|
+
)) {
|
|
224
231
|
console.log(
|
|
225
232
|
`Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=expired and source not expired)`
|
|
226
233
|
);
|
|
@@ -442,15 +449,15 @@ var SQLiteStore = class {
|
|
|
442
449
|
const row = stmt.get(sourceId);
|
|
443
450
|
return Boolean(row);
|
|
444
451
|
}
|
|
445
|
-
sourceExpired(sourceId) {
|
|
452
|
+
sourceExpired(sourceId, now) {
|
|
446
453
|
const stmt = this.#db.prepare(dedent`
|
|
447
454
|
SELECT 1 FROM sources
|
|
448
455
|
WHERE source_id = ?
|
|
449
456
|
AND expires_at IS NOT NULL
|
|
450
|
-
AND expires_at <=
|
|
457
|
+
AND expires_at <= ?
|
|
451
458
|
LIMIT 1
|
|
452
459
|
`);
|
|
453
|
-
const row = stmt.get(sourceId);
|
|
460
|
+
const row = stmt.get(sourceId, now.toISOString());
|
|
454
461
|
return Boolean(row);
|
|
455
462
|
}
|
|
456
463
|
setSourceExpiry(sourceId, expiryDate) {
|
package/dist/index.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/lib/embedders/huggingface.ts", "../src/lib/embedders/fastembed.ts", "../src/lib/ingest.ts", "../src/lib/stores/cid.ts", "../src/lib/similiarty-search.ts", "../src/lib/stores/sqlite/open.ts", "../src/lib/stores/sqlite/sqlite.ts", "../src/lib/stores/sqlite/ddl.sqlite.ts", "../src/lib/stores/sqlite/node-sqlite.ts"],
|
|
4
|
-
"sourcesContent": ["import type {\n FeatureExtractionPipeline,\n Tensor,\n} from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.ts';\n\nexport type FeatureExtractionFn = FeatureExtractionPipeline;\n\nexport interface EmbeddingOptions {\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline;\n pooling?: 'mean' | 'cls';\n normalize?: boolean;\n}\n\nasync function textEmbeddings(\n inputs: string | string[],\n { extractorFn, pooling, normalize }: EmbeddingOptions,\n) {\n inputs = (Array.isArray(inputs) ? inputs : [inputs]).map((it) => it.trim());\n const extractor = await extractorFn();\n const tensor = await extractor(inputs, {\n pooling: pooling ?? 'mean',\n normalize: normalize ?? true,\n });\n return tensorToEmbeddings(tensor);\n}\n\nexport function tensorToEmbeddings(tensor: Tensor) {\n const dims = tensor.dims;\n if (!Array.isArray(dims) || dims.length < 2) {\n throw new Error(`Unexpected tensor dims: ${JSON.stringify(dims)}`);\n }\n const batchSize = dims[0];\n const hiddenSize = dims[dims.length - 1];\n const expectedLen = batchSize * hiddenSize;\n if (tensor.data.length !== expectedLen) {\n throw new Error(\n `Data length mismatch: got ${tensor.data.length}, expected ${expectedLen} (batch=${batchSize}, hidden=${hiddenSize})`,\n );\n }\n const data = tensor.data;\n if (!(data instanceof Float32Array)) {\n throw new Error(`Expected float32 embeddings, got ${tensor.type}`);\n }\n // Reuse the underlying typed array without copying; we'll copy when creating Buffer for SQLite\n const embeddings: Float32Array[] = [];\n const dimensions = tensor.dims[tensor.dims.length - 1];\n for (let i = 0; i < batchSize; i++) {\n const start = i * hiddenSize;\n embeddings.push(data.subarray(start, start + hiddenSize));\n }\n return { embeddings, dimensions };\n}\n\nexport function huggingface(\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline,\n): Embedder {\n return (documents) => textEmbeddings(documents, { extractorFn });\n}\n", "import type { FeatureExtractionPipeline } from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.ts';\nimport { tensorToEmbeddings } from './huggingface.ts';\n\n/**\n * Stable model identifiers retained from the former `fastembed` adapter.\n */\nexport const EmbeddingModels = {\n AllMiniLML6V2: 'fast-all-MiniLM-L6-v2',\n BGEBaseEN: 'fast-bge-base-en',\n BGEBaseENV15: 'fast-bge-base-en-v1.5',\n BGESmallEN: 'fast-bge-small-en',\n BGESmallENV15: 'fast-bge-small-en-v1.5',\n BGESmallZH: 'fast-bge-small-zh-v1.5',\n MLE5Large: 'fast-multilingual-e5-large',\n} as const;\nexport type EmbeddingModel =\n (typeof EmbeddingModels)[keyof typeof EmbeddingModels];\n\nconst transformerModels: Record<EmbeddingModel, string> = {\n [EmbeddingModels.AllMiniLML6V2]: 'Xenova/all-MiniLM-L6-v2',\n [EmbeddingModels.BGEBaseEN]: 'Xenova/bge-base-en',\n [EmbeddingModels.BGEBaseENV15]: 'Xenova/bge-base-en-v1.5',\n [EmbeddingModels.BGESmallEN]: 'Xenova/bge-small-en',\n [EmbeddingModels.BGESmallENV15]: 'Xenova/bge-small-en-v1.5',\n [EmbeddingModels.BGESmallZH]: 'Xenova/bge-small-zh-v1.5',\n [EmbeddingModels.MLE5Large]: 'Xenova/multilingual-e5-large',\n};\n\nexport interface FastEmbedOptions {\n model?: EmbeddingModel;\n batchSize?: number;\n cacheDir?: string;\n}\n\n/**\n * Local feature extraction backed by Transformers.js. The historical function\n * and model identifiers remain stable while avoiding FastEmbed's archived tar\n * extraction dependency.\n */\nexport function fastembed(options: FastEmbedOptions = {}): Embedder {\n const {\n model: modelId = EmbeddingModels.BGESmallENV15,\n batchSize = 256,\n cacheDir,\n } = options;\n\n if (!Number.isInteger(batchSize) || batchSize <= 0) {\n throw new RangeError('batchSize must be a positive integer');\n }\n\n let modelPromise: Promise<FeatureExtractionPipeline> | null = null;\n const getModel = () => {\n if (!modelPromise) {\n modelPromise = import('@huggingface/transformers').then(({ pipeline }) =>\n pipeline('feature-extraction', transformerModels[modelId], {\n cache_dir: cacheDir,\n }),\n );\n }\n return modelPromise;\n };\n\n return async (documents: string[]) => {\n if (documents.length === 0) {\n return { embeddings: [], dimensions: 0 };\n }\n\n const model = await getModel();\n const embeddings: Float32Array[] = [];\n let dimensions = 0;\n\n for (let start = 0; start < documents.length; start += batchSize) {\n const tensor = await model(documents.slice(start, start + batchSize), {\n pooling: 'mean',\n normalize: true,\n });\n const batch = tensorToEmbeddings(tensor);\n dimensions ||= batch.dimensions;\n embeddings.push(...batch.embeddings);\n }\n\n return { embeddings, dimensions };\n };\n}\n", "import {\n MarkdownTextSplitter,\n RecursiveCharacterTextSplitter,\n} from '@langchain/textsplitters';\n\nimport type { Connector } from './connectors/connector.ts';\nimport type { Splitter } from './pipeline.ts';\nimport { cid } from './stores/cid.ts';\nimport type { Embedder, Store } from './stores/store.ts';\n\nexport interface IngestionConfig {\n connector: Connector;\n store: Store;\n splitter?: Splitter;\n embedder: Embedder;\n}\n\nexport async function ingest(\n config: IngestionConfig,\n callback?: (it: string) => void,\n) {\n const splitter = config.splitter ?? split;\n const embedder = config.embedder;\n const corpuses = config.connector.sources();\n const { expiresAfter } = config.connector;\n const expiryDate =\n expiresAfter === undefined\n ? undefined\n : new Date(Date.now() + expiresAfter);\n\n for await (const it of corpuses) {\n callback?.(it.id);\n const content = await it.content();\n if (!content.trim()) {\n // skip empty files\n continue;\n }\n await config.store.index(\n config.connector.sourceId,\n {\n id: it.id,\n cid: cid(content),\n metadata: it.metadata,\n chunker: async function* () {\n // Embed in small batches to control memory usage\n const values = await splitter(it.id, content);\n const batchSize = 40;\n for (let i = 0; i < values.length; i += batchSize) {\n const batch = values.slice(i, i + batchSize);\n const { embeddings } = await embedder(batch);\n for (let j = 0; j < embeddings.length; j++) {\n yield {\n content: batch[j],\n embedding: embeddings[j],\n };\n }\n }\n },\n },\n expiryDate,\n );\n }\n}\n\nfunction split(id: string, content: string) {\n const splitter = new MarkdownTextSplitter();\n return splitter.splitText(content);\n}\n\nexport type ChunkPosition = {\n startLine: number;\n startColumn: number;\n endLine: number;\n endColumn: number;\n};\n\nexport type SplitChunkWithPosition = {\n content: string;\n position: ChunkPosition | null;\n index: number;\n};\n\nfunction normalizeNewlines(value: string) {\n return value.replace(/\\r\\n/g, '\\n');\n}\n\nfunction computePositions(\n originalContent: string,\n chunks: string[],\n): Array<ChunkPosition | null> {\n if (!chunks.length) {\n return [];\n }\n\n const normalizedContent = normalizeNewlines(originalContent);\n const positions: Array<ChunkPosition | null> = [];\n let searchOffset = 0;\n\n for (const chunk of chunks) {\n const normalizedChunk = normalizeNewlines(chunk);\n const trimmedChunk = normalizedChunk.trim();\n\n const seek = (needle: string, fromIndex: number) =>\n needle ? normalizedContent.indexOf(needle, fromIndex) : -1;\n\n let matchIndex = seek(normalizedChunk, searchOffset);\n let matchValue = normalizedChunk;\n\n if (matchIndex === -1 && trimmedChunk) {\n matchIndex = seek(trimmedChunk, searchOffset);\n matchValue = trimmedChunk;\n }\n\n if (matchIndex === -1) {\n matchIndex = seek(normalizedChunk, 0);\n matchValue = normalizedChunk;\n }\n\n if (matchIndex === -1 && trimmedChunk) {\n matchIndex = seek(trimmedChunk, 0);\n matchValue = trimmedChunk;\n }\n\n if (matchIndex === -1) {\n positions.push(null);\n continue;\n }\n\n const before = normalizedContent.slice(0, matchIndex);\n const beforeLines = before.split('\\n');\n const startLine = beforeLines.length;\n const startColumn = beforeLines[beforeLines.length - 1].length + 1;\n\n const lines = matchValue.split('\\n');\n const endLine = startLine + lines.length - 1;\n const endColumn =\n lines.length === 1\n ? startColumn + lines[0].length\n : lines[lines.length - 1].length + 1;\n\n positions.push({ startLine, startColumn, endLine, endColumn });\n searchOffset = matchIndex + matchValue.length;\n }\n\n return positions;\n}\n\nfunction buildChunksWithPositions(\n originalContent: string,\n chunks: string[],\n): SplitChunkWithPosition[] {\n const positions = computePositions(originalContent, chunks);\n return chunks.map((content, index) => ({\n content,\n index,\n position: positions[index] ?? null,\n }));\n}\n\nexport async function splitTypeScriptWithPositions(\n id: string,\n content: string,\n): Promise<SplitChunkWithPosition[]> {\n const splitter = RecursiveCharacterTextSplitter.fromLanguage('js', {\n chunkSize: 512,\n chunkOverlap: 100,\n });\n const docs = await splitter.createDocuments([content]);\n const chunks = docs.map((d) => d.pageContent);\n return buildChunksWithPositions(content, chunks);\n}\n\nexport async function splitTypeScript(id: string, content: string) {\n const chunks = await splitTypeScriptWithPositions(id, content);\n return chunks.map((chunk) => chunk.content);\n}\n", "import { createHash } from 'node:crypto';\n\nexport function cid(content: string) {\n return createHash('sha256').update(content).digest('hex');\n}\n", "import { embedMany } from 'ai';\n\nimport { type IngestionConfig, ingest } from './ingest.ts';\n\nexport async function similaritySearch(\n query: string,\n config: Omit<IngestionConfig, 'splitter'>,\n) {\n // Ingest if needed, then perform vector similarity search via the configured store\n const mode = config.connector.ingestWhen ?? 'contentChanged';\n let shouldIngest = true;\n\n if (mode === 'never') {\n if (await config.store.sourceExists(config.connector.sourceId)) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=never and source exists)`,\n );\n shouldIngest = false;\n }\n } else if (mode === 'expired') {\n const sourceExists = await config.store.sourceExists(\n config.connector.sourceId,\n );\n if (\n sourceExists &&\n !(await config.store.sourceExpired(config.connector.sourceId))\n ) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=expired and source not expired)`,\n );\n shouldIngest = false;\n }\n }\n\n if (shouldIngest) {\n await ingest(config);\n }\n return config.store\n .search(\n query,\n { sourceId: config.connector.sourceId, topN: 50 },\n config.embedder,\n )\n .then((results) =>\n results.map((it) => ({\n ...it,\n similarity: 1 - it.distance,\n distance: it.distance,\n })),\n );\n}\n", "import { WriteAheadLog, open } from '@zukhruf/sqlite-open';\nimport * as sqliteVec from 'sqlite-vec';\n\n// The store's statements use the vec0 module, so every connection that runs\n// them, the dev database of the declaration included, loads sqlite-vec. The\n// rows of vectors are large, so a new file gets 32 KiB pages; the write-ahead\n// log fixes the page size, so the journal sets it before it.\nexport function openWithVectors(path: string) {\n const db = open(path, {\n allowExtension: true,\n journal: new WriteAheadLog({ pageSize: 32768 }),\n });\n db.loadExtension(sqliteVec.getLoadablePath());\n return db;\n}\n", "import { Declaration } from '@zukhruf/sqlite-drift';\nimport { transaction } from '@zukhruf/sqlite-transaction';\nimport dedent from 'dedent';\nimport type { DatabaseSync } from 'node:sqlite';\n\nimport type {\n Chunk,\n Corpus,\n Embedder,\n SearchOptions,\n SearchResult,\n Store,\n} from '../store.ts';\nimport { SETUP, statements, vectors } from './ddl.sqlite.ts';\nimport { openWithVectors } from './open.ts';\n\nconst DEFAULT_TOP_N = 10;\n\nconst tables = new Declaration(statements, { rows: 'keep' });\n\nexport class SQLiteStore implements Store {\n readonly #db: DatabaseSync;\n constructor(db: DatabaseSync, dimension: number) {\n this.#db = db;\n this.#db.exec(SETUP);\n const chunks = new Declaration([vectors(dimension)], {\n rows: 'replace',\n devDatabase: () => openWithVectors(':memory:'),\n });\n transaction(\n this.#db,\n () => {\n const stale = chunks.drift(this.#db) === 'stale';\n chunks.apply(this.#db);\n // A replaced vec_chunks is empty, yet upsertDoc skips a document whose\n // cid is unchanged and similaritySearch skips a source that exists.\n // Deleting the sources cascades to their documents, so the next\n // ingest embeds every document again.\n if (stale) this.#db.exec('DELETE FROM sources');\n },\n { mode: 'immediate' },\n );\n tables.apply(this.#db);\n }\n\n #searchByDocument(inputs: {\n sourceId: string;\n documentId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.document_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(\n inputs.sourceId,\n inputs.documentId,\n inputs.embedding,\n inputs.k,\n );\n }\n #searchBySource(inputs: {\n sourceId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(inputs.sourceId, inputs.embedding, inputs.k);\n }\n async search(query: string, options: SearchOptions, embedder: Embedder) {\n const { embeddings } = await embedder([query]);\n if (!embeddings.length) {\n return [];\n }\n const vectorBlob = vectorToBlob(embeddings[0]);\n const topN = options.topN;\n\n if (options.documentId) {\n const rows = this.#searchByDocument({\n sourceId: options.sourceId,\n documentId: options.documentId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n\n const rows = this.#searchBySource({\n sourceId: options.sourceId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n #upsertSource(inputs: { sourceId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id) VALUES (?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t`);\n return stmt.run(inputs.sourceId);\n }\n #upsertSourceWithExpiry(inputs: { sourceId: string; expiresAt: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id, expires_at) VALUES (?, ?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET\n\t\t\t\t\tupdated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now'),\n\t\t\t\t\texpires_at=excluded.expires_at\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.expiresAt);\n }\n upsertDoc(inputs: {\n documentId: string;\n sourceId: string;\n cid: string;\n metadata?: Record<string, any>;\n }) {\n const stmt = this.#db.prepare(dedent`\n INSERT INTO documents (id, source_id, cid, metadata)\n VALUES (?, ?, ?, json(?))\n ON CONFLICT(id) DO UPDATE SET\n cid=excluded.cid,\n metadata=COALESCE(excluded.metadata, documents.metadata),\n updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n WHERE documents.cid != excluded.cid;\n `);\n return stmt.run(\n inputs.documentId,\n inputs.sourceId,\n inputs.cid,\n inputs.metadata ? JSON.stringify(inputs.metadata) : null,\n );\n }\n insertDoc(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO vec_chunks (source_id, document_id, content, embedding)\n\t\t\t\tVALUES (?, ?, ?, vec_normalize(vec_f32(?)))\n\t\t\t`);\n return (chunk: Chunk) => {\n stmt.run(\n inputs.sourceId,\n inputs.documentId,\n chunk.content,\n vectorToBlob(chunk.embedding),\n );\n };\n }\n delete(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tDELETE FROM vec_chunks WHERE source_id = ? AND document_id = ?\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.documentId);\n }\n sourceExists(sourceId: string) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources WHERE source_id = ? LIMIT 1\n\t\t\t`);\n const row = stmt.get(sourceId);\n return Boolean(row);\n }\n sourceExpired(sourceId: string) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources\n\t\t\t\tWHERE source_id = ?\n\t\t\t\t\tAND expires_at IS NOT NULL\n\t\t\t\t\tAND expires_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tLIMIT 1\n\t\t\t`);\n const row = stmt.get(sourceId);\n return Boolean(row);\n }\n setSourceExpiry(sourceId: string, expiryDate: Date) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tUPDATE sources\n\t\t\t\tSET expires_at = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tWHERE source_id = ?\n\t\t\t`);\n stmt.run(expiryDate.toISOString(), sourceId);\n }\n async index(sourceId: string, corpus: Corpus, expiryDate?: Date) {\n if (expiryDate) {\n this.#upsertSourceWithExpiry({\n expiresAt: expiryDate.toISOString(),\n sourceId,\n });\n } else {\n this.#upsertSource({ sourceId });\n }\n\n const info = this.upsertDoc({\n documentId: corpus.id,\n sourceId,\n cid: corpus.cid,\n metadata: corpus.metadata,\n });\n const changed = info.changes > 0;\n\n if (!changed) {\n return;\n }\n\n const insert = this.insertDoc({ sourceId, documentId: corpus.id });\n // Delete previous rows once before inserting\n transaction(\n this.#db,\n () => {\n this.delete({ sourceId, documentId: corpus.id });\n },\n { mode: 'immediate' },\n );\n\n const batchSize = 32;\n let batch: Chunk[] = [];\n const flush = () => {\n if (!batch.length) return;\n transaction(\n this.#db,\n () => {\n for (let i = 0; i < batch.length; i++) {\n insert(batch[i]);\n }\n },\n { mode: 'immediate' },\n );\n batch = [];\n };\n\n for await (const chunk of corpus.chunker()) {\n batch.push(chunk);\n if (batch.length >= batchSize) flush();\n }\n // flush any remaining\n flush();\n }\n}\n\nexport function vectorToBlob(vector: number[] | Float32Array): Buffer {\n if (vector instanceof Float32Array) {\n // Copy into a fresh Buffer to avoid retaining references to a larger batch tensor buffer\n const copied = new Float32Array(vector.length);\n copied.set(vector);\n return Buffer.from(copied.buffer);\n }\n const floatArray = new Float32Array(vector);\n return Buffer.from(floatArray.buffer);\n}\n\nfunction toSearchResult(row: unknown): SearchResult {\n if (\n typeof row !== 'object' ||\n row === null ||\n !('content' in row) ||\n typeof row.content !== 'string' ||\n !('distance' in row) ||\n typeof row.distance !== 'number' ||\n !('document_id' in row) ||\n typeof row.document_id !== 'string' ||\n !('metadata' in row) ||\n (row.metadata !== null && typeof row.metadata !== 'string')\n ) {\n throw new Error('Unexpected vector search row');\n }\n return {\n content: row.content,\n distance: row.distance,\n document_id: row.document_id,\n metadata: parseMetadata(row.metadata),\n };\n}\n\nfunction parseMetadata(value: string | null): Record<string, unknown> | null {\n if (value === null) return null;\n try {\n const parsed: unknown = JSON.parse(value);\n return isRecord(parsed) ? parsed : null;\n } catch {\n return null;\n }\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n", "// Embedding store schema for SQLite. SQLite stores the text of each CREATE and\n// @zukhruf/sqlite-drift compares it, so the text after each object name must\n// stay byte-identical to what earlier releases wrote, inline comments included.\n\n// Session-level PRAGMAs, run on each connection before the schema is applied.\nexport const SETUP = `\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n`;\n\nexport function vectors(dimension: number) {\n return `CREATE VIRTUAL TABLE vec_chunks USING vec0(\n source_id TEXT PARTITION KEY,\n document_id TEXT,\n embedding FLOAT[${dimension}] DISTANCE_METRIC=cosine,\n +content TEXT -- auxiliary payload (not filterable) for reconstruction\n)`;\n}\n\nexport const statements = [\n `CREATE TABLE sources (\n source_id TEXT PRIMARY KEY,\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n expires_at TEXT\n) STRICT`,\n `CREATE TABLE documents (\n id TEXT PRIMARY KEY,\n source_id TEXT NOT NULL,\n cid TEXT NOT NULL,\n metadata TEXT, -- JSON blob with arbitrary document metadata\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT`,\n // Indexes to accelerate lookups / maintenance\n `CREATE INDEX idx_documents_source ON documents(source_id)`,\n `CREATE INDEX idx_documents_source_updated ON documents(source_id, updated_at)`,\n `CREATE INDEX idx_documents_cid ON documents(cid)`,\n `CREATE INDEX idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL`,\n // emulate cascade for the virtual table\n `CREATE TRIGGER trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n DELETE FROM vec_chunks\n WHERE source_id = OLD.source_id\n AND document_id = OLD.id;\nEND`,\n];\n", "import { openWithVectors } from './open.ts';\nimport { SQLiteStore } from './sqlite.ts';\n\nexport function nodeSQLite(dbName: string, dimension: number) {\n return new SQLiteStore(openWithVectors(dbName), dimension);\n}\n"],
|
|
5
|
-
"mappings": ";AA6BO,SAAS,mBAAmB,QAAgB;AACjD,QAAM,OAAO,OAAO;AACpB,MAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,SAAS,GAAG;AAC3C,UAAM,IAAI,MAAM,2BAA2B,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,EACnE;AACA,QAAM,YAAY,KAAK,CAAC;AACxB,QAAM,aAAa,KAAK,KAAK,SAAS,CAAC;AACvC,QAAM,cAAc,YAAY;AAChC,MAAI,OAAO,KAAK,WAAW,aAAa;AACtC,UAAM,IAAI;AAAA,MACR,6BAA6B,OAAO,KAAK,MAAM,cAAc,WAAW,WAAW,SAAS,YAAY,UAAU;AAAA,IACpH;AAAA,EACF;AACA,QAAM,OAAO,OAAO;AACpB,MAAI,EAAE,gBAAgB,eAAe;AACnC,UAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AAAA,EACnE;AAEA,QAAM,aAA6B,CAAC;AACpC,QAAM,aAAa,OAAO,KAAK,OAAO,KAAK,SAAS,CAAC;AACrD,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,IAAI;AAClB,eAAW,KAAK,KAAK,SAAS,OAAO,QAAQ,UAAU,CAAC;AAAA,EAC1D;AACA,SAAO,EAAE,YAAY,WAAW;AAClC;;;AC9CO,IAAM,kBAAkB;AAAA,EAC7B,eAAe;AAAA,EACf,WAAW;AAAA,EACX,cAAc;AAAA,EACd,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,WAAW;AACb;AAIA,IAAM,oBAAoD;AAAA,EACxD,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,SAAS,GAAG;AAAA,EAC7B,CAAC,gBAAgB,YAAY,GAAG;AAAA,EAChC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,SAAS,GAAG;AAC/B;AAaO,SAAS,UAAU,UAA4B,CAAC,GAAa;AAClE,QAAM;AAAA,IACJ,OAAO,UAAU,gBAAgB;AAAA,IACjC,YAAY;AAAA,IACZ;AAAA,EACF,IAAI;AAEJ,MAAI,CAAC,OAAO,UAAU,SAAS,KAAK,aAAa,GAAG;AAClD,UAAM,IAAI,WAAW,sCAAsC;AAAA,EAC7D;AAEA,MAAI,eAA0D;AAC9D,QAAM,WAAW,MAAM;AACrB,QAAI,CAAC,cAAc;AACjB,qBAAe,OAAO,2BAA2B,EAAE;AAAA,QAAK,CAAC,EAAE,SAAS,MAClE,SAAS,sBAAsB,kBAAkB,OAAO,GAAG;AAAA,UACzD,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,SAAO,OAAO,cAAwB;AACpC,QAAI,UAAU,WAAW,GAAG;AAC1B,aAAO,EAAE,YAAY,CAAC,GAAG,YAAY,EAAE;AAAA,IACzC;AAEA,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,aAA6B,CAAC;AACpC,QAAI,aAAa;AAEjB,aAAS,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,WAAW;AAChE,YAAM,SAAS,MAAM,MAAM,UAAU,MAAM,OAAO,QAAQ,SAAS,GAAG;AAAA,QACpE,SAAS;AAAA,QACT,WAAW;AAAA,MACb,CAAC;AACD,YAAM,QAAQ,mBAAmB,MAAM;AACvC,qBAAe,MAAM;AACrB,iBAAW,KAAK,GAAG,MAAM,UAAU;AAAA,IACrC;AAEA,WAAO,EAAE,YAAY,WAAW;AAAA,EAClC;AACF;;;ACrFA;AAAA,EACE;AAAA,EACA;AAAA,OACK;;;ACHP,SAAS,kBAAkB;AAEpB,SAAS,IAAI,SAAiB;AACnC,SAAO,WAAW,QAAQ,EAAE,OAAO,OAAO,EAAE,OAAO,KAAK;AAC1D;;;ADaA,eAAsB,OACpB,QACA,UACA;AACA,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,WAAW,OAAO;AACxB,QAAM,WAAW,OAAO,UAAU,QAAQ;AAC1C,QAAM,EAAE,aAAa,IAAI,OAAO;AAChC,QAAM,aACJ,iBAAiB,SACb,SACA,IAAI,KAAK,KAAK,IAAI,IAAI,YAAY;AAExC,mBAAiB,MAAM,UAAU;AAC/B,eAAW,GAAG,EAAE;AAChB,UAAM,UAAU,MAAM,GAAG,QAAQ;AACjC,QAAI,CAAC,QAAQ,KAAK,GAAG;AAEnB;AAAA,IACF;AACA,UAAM,OAAO,MAAM;AAAA,MACjB,OAAO,UAAU;AAAA,MACjB;AAAA,QACE,IAAI,GAAG;AAAA,QACP,KAAK,IAAI,OAAO;AAAA,QAChB,UAAU,GAAG;AAAA,QACb,SAAS,mBAAmB;AAE1B,gBAAM,SAAS,MAAM,SAAS,GAAG,IAAI,OAAO;AAC5C,gBAAM,YAAY;AAClB,mBAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,WAAW;AACjD,kBAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,SAAS;AAC3C,kBAAM,EAAE,WAAW,IAAI,MAAM,SAAS,KAAK;AAC3C,qBAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,oBAAM;AAAA,gBACJ,SAAS,MAAM,CAAC;AAAA,gBAChB,WAAW,WAAW,CAAC;AAAA,cACzB;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,MAAM,IAAY,SAAiB;AAC1C,QAAM,WAAW,IAAI,qBAAqB;AAC1C,SAAO,SAAS,UAAU,OAAO;AACnC;AAeA,SAAS,kBAAkB,OAAe;AACxC,SAAO,MAAM,QAAQ,SAAS,IAAI;AACpC;AAEA,SAAS,iBACP,iBACA,QAC6B;AAC7B,MAAI,CAAC,OAAO,QAAQ;AAClB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,kBAAkB,eAAe;AAC3D,QAAM,YAAyC,CAAC;AAChD,MAAI,eAAe;AAEnB,aAAW,SAAS,QAAQ;AAC1B,UAAM,kBAAkB,kBAAkB,KAAK;AAC/C,UAAM,eAAe,gBAAgB,KAAK;AAE1C,UAAM,OAAO,CAAC,QAAgB,cAC5B,SAAS,kBAAkB,QAAQ,QAAQ,SAAS,IAAI;AAE1D,QAAI,aAAa,KAAK,iBAAiB,YAAY;AACnD,QAAI,aAAa;AAEjB,QAAI,eAAe,MAAM,cAAc;AACrC,mBAAa,KAAK,cAAc,YAAY;AAC5C,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,IAAI;AACrB,mBAAa,KAAK,iBAAiB,CAAC;AACpC,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,MAAM,cAAc;AACrC,mBAAa,KAAK,cAAc,CAAC;AACjC,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,IAAI;AACrB,gBAAU,KAAK,IAAI;AACnB;AAAA,IACF;AAEA,UAAM,SAAS,kBAAkB,MAAM,GAAG,UAAU;AACpD,UAAM,cAAc,OAAO,MAAM,IAAI;AACrC,UAAM,YAAY,YAAY;AAC9B,UAAM,cAAc,YAAY,YAAY,SAAS,CAAC,EAAE,SAAS;AAEjE,UAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,UAAM,UAAU,YAAY,MAAM,SAAS;AAC3C,UAAM,YACJ,MAAM,WAAW,IACb,cAAc,MAAM,CAAC,EAAE,SACvB,MAAM,MAAM,SAAS,CAAC,EAAE,SAAS;AAEvC,cAAU,KAAK,EAAE,WAAW,aAAa,SAAS,UAAU,CAAC;AAC7D,mBAAe,aAAa,WAAW;AAAA,EACzC;AAEA,SAAO;AACT;AAEA,SAAS,yBACP,iBACA,QAC0B;AAC1B,QAAM,YAAY,iBAAiB,iBAAiB,MAAM;AAC1D,SAAO,OAAO,IAAI,CAAC,SAAS,WAAW;AAAA,IACrC;AAAA,IACA;AAAA,IACA,UAAU,UAAU,KAAK,KAAK;AAAA,EAChC,EAAE;AACJ;AAEA,eAAsB,6BACpB,IACA,SACmC;AACnC,QAAM,WAAW,+BAA+B,aAAa,MAAM;AAAA,IACjE,WAAW;AAAA,IACX,cAAc;AAAA,EAChB,CAAC;AACD,QAAM,OAAO,MAAM,SAAS,gBAAgB,CAAC,OAAO,CAAC;AACrD,QAAM,SAAS,KAAK,IAAI,CAAC,MAAM,EAAE,WAAW;AAC5C,SAAO,yBAAyB,SAAS,MAAM;AACjD;AAEA,eAAsB,gBAAgB,IAAY,SAAiB;AACjE,QAAM,SAAS,MAAM,6BAA6B,IAAI,OAAO;AAC7D,SAAO,OAAO,IAAI,CAAC,UAAU,MAAM,OAAO;AAC5C;;;AE/KA,OAA0B;AAI1B,eAAsB,iBACpB,OACA,QACA;AAEA,QAAM,OAAO,OAAO,UAAU,cAAc;AAC5C,MAAI,eAAe;AAEnB,MAAI,SAAS,SAAS;AACpB,QAAI,MAAM,OAAO,MAAM,aAAa,OAAO,UAAU,QAAQ,GAAG;AAC9D,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF,WAAW,SAAS,WAAW;AAC7B,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,IACnB;AACA,QACE,gBACA,CAAE,MAAM,OAAO,MAAM,cAAc,OAAO,UAAU,QAAQ,GAC5D;AACA,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAEA,MAAI,cAAc;AAChB,UAAM,OAAO,MAAM;AAAA,EACrB;AACA,SAAO,OAAO,MACX;AAAA,IACC;AAAA,IACA,EAAE,UAAU,OAAO,UAAU,UAAU,MAAM,GAAG;AAAA,IAChD,OAAO;AAAA,EACT,EACC;AAAA,IAAK,CAAC,YACL,QAAQ,IAAI,CAAC,QAAQ;AAAA,MACnB,GAAG;AAAA,MACH,YAAY,IAAI,GAAG;AAAA,MACnB,UAAU,GAAG;AAAA,IACf,EAAE;AAAA,EACJ;AACJ;;;AClDA,SAAS,eAAe,YAAY;AACpC,YAAY,eAAe;AAMpB,SAAS,gBAAgB,MAAc;AAC5C,QAAM,KAAK,KAAK,MAAM;AAAA,IACpB,gBAAgB;AAAA,IAChB,SAAS,IAAI,cAAc,EAAE,UAAU,MAAM,CAAC;AAAA,EAChD,CAAC;AACD,KAAG,cAAwB,0BAAgB,CAAC;AAC5C,SAAO;AACT;;;ACdA,SAAS,mBAAmB;AAC5B,SAAS,mBAAmB;AAC5B,OAAO,YAAY;;;ACGZ,IAAM,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AASd,SAAS,QAAQ,WAAmB;AACzC,SAAO;AAAA;AAAA;AAAA,oBAGW,SAAS;AAAA;AAAA;AAG7B;AAEO,IAAM,aAAa;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA;AAAA,EAKA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOF;;;ADlCA,IAAM,gBAAgB;AAEtB,IAAM,SAAS,IAAI,YAAY,YAAY,EAAE,MAAM,OAAO,CAAC;AAEpD,IAAM,cAAN,MAAmC;AAAA,EAC/B;AAAA,EACT,YAAY,IAAkB,WAAmB;AAC/C,SAAK,MAAM;AACX,SAAK,IAAI,KAAK,KAAK;AACnB,UAAM,SAAS,IAAI,YAAY,CAAC,QAAQ,SAAS,CAAC,GAAG;AAAA,MACnD,MAAM;AAAA,MACN,aAAa,MAAM,gBAAgB,UAAU;AAAA,IAC/C,CAAC;AACD;AAAA,MACE,KAAK;AAAA,MACL,MAAM;AACJ,cAAM,QAAQ,OAAO,MAAM,KAAK,GAAG,MAAM;AACzC,eAAO,MAAM,KAAK,GAAG;AAKrB,YAAI,MAAO,MAAK,IAAI,KAAK,qBAAqB;AAAA,MAChD;AAAA,MACA,EAAE,MAAM,YAAY;AAAA,IACtB;AACA,WAAO,MAAM,KAAK,GAAG;AAAA,EACvB;AAAA,EAEA,kBAAkB,QAKf;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAS9B;AACA,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,gBAAgB,QAIb;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQ9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,WAAW,OAAO,CAAC;AAAA,EAC7D;AAAA,EACA,MAAM,OAAO,OAAe,SAAwB,UAAoB;AACtE,UAAM,EAAE,WAAW,IAAI,MAAM,SAAS,CAAC,KAAK,CAAC;AAC7C,QAAI,CAAC,WAAW,QAAQ;AACtB,aAAO,CAAC;AAAA,IACV;AACA,UAAM,aAAa,aAAa,WAAW,CAAC,CAAC;AAC7C,UAAM,OAAO,QAAQ;AAErB,QAAI,QAAQ,YAAY;AACtB,YAAMA,QAAO,KAAK,kBAAkB;AAAA,QAClC,UAAU,QAAQ;AAAA,QAClB,YAAY,QAAQ;AAAA,QACpB,WAAW;AAAA,QACX,GAAG,QAAQ;AAAA,MACb,CAAC;AACD,aAAOA,MAAK,IAAI,cAAc;AAAA,IAChC;AAEA,UAAM,OAAO,KAAK,gBAAgB;AAAA,MAChC,UAAU,QAAQ;AAAA,MAClB,WAAW;AAAA,MACX,GAAG,QAAQ;AAAA,IACb,CAAC;AACD,WAAO,KAAK,IAAI,cAAc;AAAA,EAChC;AAAA,EACA,cAAc,QAA8B;AAC1C,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,KAAK,IAAI,OAAO,QAAQ;AAAA,EACjC;AAAA,EACA,wBAAwB,QAAiD;AACvE,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAK9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,SAAS;AAAA,EACnD;AAAA,EACA,UAAU,QAKP;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,OAQ3B;AACH,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO,WAAW,KAAK,UAAU,OAAO,QAAQ,IAAI;AAAA,IACtD;AAAA,EACF;AAAA,EACA,UAAU,QAAkD;AAC1D,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,CAAC,UAAiB;AACvB,WAAK;AAAA,QACH,OAAO;AAAA,QACP,OAAO;AAAA,QACP,MAAM;AAAA,QACN,aAAa,MAAM,SAAS;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAAA,EACA,OAAO,QAAkD;AACvD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,UAAU;AAAA,EACpD;AAAA,EACA,aAAa,UAAkB;AAC7B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,cAAc,UAAkB;AAC9B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAM9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,gBAAgB,UAAkB,YAAkB;AAClD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA,IAI9B;AACA,SAAK,IAAI,WAAW,YAAY,GAAG,QAAQ;AAAA,EAC7C;AAAA,EACA,MAAM,MAAM,UAAkB,QAAgB,YAAmB;AAC/D,QAAI,YAAY;AACd,WAAK,wBAAwB;AAAA,QAC3B,WAAW,WAAW,YAAY;AAAA,QAClC;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,WAAK,cAAc,EAAE,SAAS,CAAC;AAAA,IACjC;AAEA,UAAM,OAAO,KAAK,UAAU;AAAA,MAC1B,YAAY,OAAO;AAAA,MACnB;AAAA,MACA,KAAK,OAAO;AAAA,MACZ,UAAU,OAAO;AAAA,IACnB,CAAC;AACD,UAAM,UAAU,KAAK,UAAU;AAE/B,QAAI,CAAC,SAAS;AACZ;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,UAAU,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAEjE;AAAA,MACE,KAAK;AAAA,MACL,MAAM;AACJ,aAAK,OAAO,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAAA,MACjD;AAAA,MACA,EAAE,MAAM,YAAY;AAAA,IACtB;AAEA,UAAM,YAAY;AAClB,QAAI,QAAiB,CAAC;AACtB,UAAM,QAAQ,MAAM;AAClB,UAAI,CAAC,MAAM,OAAQ;AACnB;AAAA,QACE,KAAK;AAAA,QACL,MAAM;AACJ,mBAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,mBAAO,MAAM,CAAC,CAAC;AAAA,UACjB;AAAA,QACF;AAAA,QACA,EAAE,MAAM,YAAY;AAAA,MACtB;AACA,cAAQ,CAAC;AAAA,IACX;AAEA,qBAAiB,SAAS,OAAO,QAAQ,GAAG;AAC1C,YAAM,KAAK,KAAK;AAChB,UAAI,MAAM,UAAU,UAAW,OAAM;AAAA,IACvC;AAEA,UAAM;AAAA,EACR;AACF;AAEO,SAAS,aAAa,QAAyC;AACpE,MAAI,kBAAkB,cAAc;AAElC,UAAM,SAAS,IAAI,aAAa,OAAO,MAAM;AAC7C,WAAO,IAAI,MAAM;AACjB,WAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAClC;AACA,QAAM,aAAa,IAAI,aAAa,MAAM;AAC1C,SAAO,OAAO,KAAK,WAAW,MAAM;AACtC;AAEA,SAAS,eAAe,KAA4B;AAClD,MACE,OAAO,QAAQ,YACf,QAAQ,QACR,EAAE,aAAa,QACf,OAAO,IAAI,YAAY,YACvB,EAAE,cAAc,QAChB,OAAO,IAAI,aAAa,YACxB,EAAE,iBAAiB,QACnB,OAAO,IAAI,gBAAgB,YAC3B,EAAE,cAAc,QACf,IAAI,aAAa,QAAQ,OAAO,IAAI,aAAa,UAClD;AACA,UAAM,IAAI,MAAM,8BAA8B;AAAA,EAChD;AACA,SAAO;AAAA,IACL,SAAS,IAAI;AAAA,IACb,UAAU,IAAI;AAAA,IACd,aAAa,IAAI;AAAA,IACjB,UAAU,cAAc,IAAI,QAAQ;AAAA,EACtC;AACF;AAEA,SAAS,cAAc,OAAsD;AAC3E,MAAI,UAAU,KAAM,QAAO;AAC3B,MAAI;AACF,UAAM,SAAkB,KAAK,MAAM,KAAK;AACxC,WAAO,SAAS,MAAM,IAAI,SAAS;AAAA,EACrC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;;;AErSO,SAAS,WAAW,QAAgB,WAAmB;AAC5D,SAAO,IAAI,YAAY,gBAAgB,MAAM,GAAG,SAAS;AAC3D;",
|
|
4
|
+
"sourcesContent": ["import type {\n FeatureExtractionPipeline,\n Tensor,\n} from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.ts';\n\nexport type FeatureExtractionFn = FeatureExtractionPipeline;\n\nexport interface EmbeddingOptions {\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline;\n pooling?: 'mean' | 'cls';\n normalize?: boolean;\n}\n\nasync function textEmbeddings(\n inputs: string | string[],\n { extractorFn, pooling, normalize }: EmbeddingOptions,\n) {\n inputs = (Array.isArray(inputs) ? inputs : [inputs]).map((it) => it.trim());\n const extractor = await extractorFn();\n const tensor = await extractor(inputs, {\n pooling: pooling ?? 'mean',\n normalize: normalize ?? true,\n });\n return tensorToEmbeddings(tensor);\n}\n\nexport function tensorToEmbeddings(tensor: Tensor) {\n const dims = tensor.dims;\n const [batchSize, ...rest] = Array.isArray(dims) ? dims : [];\n const hiddenSize = rest.at(-1);\n if (hiddenSize === undefined) {\n throw new Error(`Unexpected tensor dims: ${JSON.stringify(dims)}`);\n }\n const expectedLen = batchSize * hiddenSize;\n if (tensor.data.length !== expectedLen) {\n throw new Error(\n `Data length mismatch: got ${tensor.data.length}, expected ${expectedLen} (batch=${batchSize}, hidden=${hiddenSize})`,\n );\n }\n const data = tensor.data;\n if (!(data instanceof Float32Array)) {\n throw new Error(`Expected float32 embeddings, got ${tensor.type}`);\n }\n // Reuse the underlying typed array without copying; we'll copy when creating Buffer for SQLite\n const embeddings: Float32Array[] = [];\n const dimensions = hiddenSize;\n for (let i = 0; i < batchSize; i++) {\n const start = i * hiddenSize;\n embeddings.push(data.subarray(start, start + hiddenSize));\n }\n return { embeddings, dimensions };\n}\n\nexport function huggingface(\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline,\n): Embedder {\n return (documents) => textEmbeddings(documents, { extractorFn });\n}\n", "import type { FeatureExtractionPipeline } from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.ts';\nimport { tensorToEmbeddings } from './huggingface.ts';\n\n/**\n * Stable model identifiers retained from the former `fastembed` adapter.\n */\nexport const EmbeddingModels = {\n AllMiniLML6V2: 'fast-all-MiniLM-L6-v2',\n BGEBaseEN: 'fast-bge-base-en',\n BGEBaseENV15: 'fast-bge-base-en-v1.5',\n BGESmallEN: 'fast-bge-small-en',\n BGESmallENV15: 'fast-bge-small-en-v1.5',\n BGESmallZH: 'fast-bge-small-zh-v1.5',\n MLE5Large: 'fast-multilingual-e5-large',\n} as const;\nexport type EmbeddingModel =\n (typeof EmbeddingModels)[keyof typeof EmbeddingModels];\n\nconst transformerModels: Record<EmbeddingModel, string> = {\n [EmbeddingModels.AllMiniLML6V2]: 'Xenova/all-MiniLM-L6-v2',\n [EmbeddingModels.BGEBaseEN]: 'Xenova/bge-base-en',\n [EmbeddingModels.BGEBaseENV15]: 'Xenova/bge-base-en-v1.5',\n [EmbeddingModels.BGESmallEN]: 'Xenova/bge-small-en',\n [EmbeddingModels.BGESmallENV15]: 'Xenova/bge-small-en-v1.5',\n [EmbeddingModels.BGESmallZH]: 'Xenova/bge-small-zh-v1.5',\n [EmbeddingModels.MLE5Large]: 'Xenova/multilingual-e5-large',\n};\n\nexport interface FastEmbedOptions {\n model?: EmbeddingModel;\n batchSize?: number;\n cacheDir?: string;\n}\n\n/**\n * Local feature extraction backed by Transformers.js. The historical function\n * and model identifiers remain stable while avoiding FastEmbed's archived tar\n * extraction dependency.\n */\nexport function fastembed(options: FastEmbedOptions = {}): Embedder {\n const {\n model: modelId = EmbeddingModels.BGESmallENV15,\n batchSize = 256,\n cacheDir,\n } = options;\n\n if (!Number.isInteger(batchSize) || batchSize <= 0) {\n throw new RangeError('batchSize must be a positive integer');\n }\n\n let modelPromise: Promise<FeatureExtractionPipeline> | null = null;\n const getModel = () => {\n if (!modelPromise) {\n modelPromise = import('@huggingface/transformers').then(({ pipeline }) =>\n pipeline('feature-extraction', transformerModels[modelId], {\n cache_dir: cacheDir,\n }),\n );\n }\n return modelPromise;\n };\n\n return async (documents: string[]) => {\n if (documents.length === 0) {\n return { embeddings: [], dimensions: 0 };\n }\n\n const model = await getModel();\n const embeddings: Float32Array[] = [];\n let dimensions = 0;\n\n for (let start = 0; start < documents.length; start += batchSize) {\n const tensor = await model(documents.slice(start, start + batchSize), {\n pooling: 'mean',\n normalize: true,\n });\n const batch = tensorToEmbeddings(tensor);\n dimensions ||= batch.dimensions;\n embeddings.push(...batch.embeddings);\n }\n\n return { embeddings, dimensions };\n };\n}\n", "import {\n MarkdownTextSplitter,\n RecursiveCharacterTextSplitter,\n} from '@langchain/textsplitters';\n\nimport type { Connector } from './connectors/connector.ts';\nimport type { Splitter } from './pipeline.ts';\nimport { cid } from './stores/cid.ts';\nimport type { Embedder, Store } from './stores/store.ts';\n\nexport interface IngestionConfig {\n connector: Connector;\n store: Store;\n splitter?: Splitter;\n embedder: Embedder;\n}\n\nexport async function ingest(\n config: IngestionConfig,\n callback?: (it: string) => void,\n) {\n const splitter = config.splitter ?? split;\n const embedder = config.embedder;\n const corpuses = config.connector.sources();\n const expiryDate = expiry(config.connector);\n\n for await (const it of corpuses) {\n callback?.(it.id);\n const content = await it.content();\n if (!content.trim()) {\n // skip empty files\n continue;\n }\n await config.store.index(\n config.connector.sourceId,\n {\n id: it.id,\n cid: cid(content),\n metadata: it.metadata,\n chunker: async function* () {\n // Embed in small batches to control memory usage\n const values = await splitter(it.id, content);\n const batchSize = 40;\n for (let i = 0; i < values.length; i += batchSize) {\n const batch = values.slice(i, i + batchSize);\n const { embeddings } = await embedder(batch);\n for (let j = 0; j < embeddings.length; j++) {\n yield {\n content: batch[j],\n embedding: embeddings[j],\n };\n }\n }\n },\n },\n expiryDate,\n );\n }\n}\n\nfunction expiry(connector: Connector) {\n if (connector.ingestWhen !== 'expired') return undefined;\n if (connector.expiresAfter === undefined) return undefined;\n return new Date(connector.clock.Date.now() + connector.expiresAfter);\n}\n\nfunction split(id: string, content: string) {\n const splitter = new MarkdownTextSplitter();\n return splitter.splitText(content);\n}\n\nexport type ChunkPosition = {\n startLine: number;\n startColumn: number;\n endLine: number;\n endColumn: number;\n};\n\nexport type SplitChunkWithPosition = {\n content: string;\n position: ChunkPosition | null;\n index: number;\n};\n\nfunction normalizeNewlines(value: string) {\n return value.replace(/\\r\\n/g, '\\n');\n}\n\nfunction computePositions(\n originalContent: string,\n chunks: string[],\n): Array<ChunkPosition | null> {\n if (!chunks.length) {\n return [];\n }\n\n const normalizedContent = normalizeNewlines(originalContent);\n const positions: Array<ChunkPosition | null> = [];\n let searchOffset = 0;\n\n for (const chunk of chunks) {\n const normalizedChunk = normalizeNewlines(chunk);\n const trimmedChunk = normalizedChunk.trim();\n\n const seek = (needle: string, fromIndex: number) =>\n needle ? normalizedContent.indexOf(needle, fromIndex) : -1;\n\n let matchIndex = seek(normalizedChunk, searchOffset);\n let matchValue = normalizedChunk;\n\n if (matchIndex === -1 && trimmedChunk) {\n matchIndex = seek(trimmedChunk, searchOffset);\n matchValue = trimmedChunk;\n }\n\n if (matchIndex === -1) {\n matchIndex = seek(normalizedChunk, 0);\n matchValue = normalizedChunk;\n }\n\n if (matchIndex === -1 && trimmedChunk) {\n matchIndex = seek(trimmedChunk, 0);\n matchValue = trimmedChunk;\n }\n\n if (matchIndex === -1) {\n positions.push(null);\n continue;\n }\n\n const before = normalizedContent.slice(0, matchIndex);\n const beforeLines = before.split('\\n');\n const startLine = beforeLines.length;\n const startColumn = before.length - before.lastIndexOf('\\n');\n\n const lines = matchValue.split('\\n');\n const endLine = startLine + lines.length - 1;\n const endColumn =\n lines.length === 1\n ? startColumn + lines[0].length\n : matchValue.length - matchValue.lastIndexOf('\\n');\n\n positions.push({ startLine, startColumn, endLine, endColumn });\n searchOffset = matchIndex + matchValue.length;\n }\n\n return positions;\n}\n\nfunction buildChunksWithPositions(\n originalContent: string,\n chunks: string[],\n): SplitChunkWithPosition[] {\n const positions = computePositions(originalContent, chunks);\n return chunks.map((content, index) => ({\n content,\n index,\n position: positions[index] ?? null,\n }));\n}\n\nexport async function splitTypeScriptWithPositions(\n id: string,\n content: string,\n): Promise<SplitChunkWithPosition[]> {\n const splitter = RecursiveCharacterTextSplitter.fromLanguage('js', {\n chunkSize: 512,\n chunkOverlap: 100,\n });\n const docs = await splitter.createDocuments([content]);\n const chunks = docs.map((d) => d.pageContent);\n return buildChunksWithPositions(content, chunks);\n}\n\nexport async function splitTypeScript(id: string, content: string) {\n const chunks = await splitTypeScriptWithPositions(id, content);\n return chunks.map((chunk) => chunk.content);\n}\n", "import { hash } from 'node:crypto';\n\nexport function cid(content: string) {\n return hash('sha256', content);\n}\n", "import { embedMany } from 'ai';\n\nimport { type IngestionConfig, ingest } from './ingest.ts';\n\nexport async function similaritySearch(\n query: string,\n config: Omit<IngestionConfig, 'splitter'>,\n) {\n // Ingest if needed, then perform vector similarity search via the configured store\n const { connector } = config;\n let shouldIngest = true;\n\n if (connector.ingestWhen === 'never') {\n if (await config.store.sourceExists(config.connector.sourceId)) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=never and source exists)`,\n );\n shouldIngest = false;\n }\n } else if (connector.ingestWhen === 'expired') {\n const sourceExists = await config.store.sourceExists(\n config.connector.sourceId,\n );\n if (\n sourceExists &&\n !(await config.store.sourceExpired(\n config.connector.sourceId,\n new Date(connector.clock.Date.now()),\n ))\n ) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=expired and source not expired)`,\n );\n shouldIngest = false;\n }\n }\n\n if (shouldIngest) {\n await ingest(config);\n }\n return config.store\n .search(\n query,\n { sourceId: config.connector.sourceId, topN: 50 },\n config.embedder,\n )\n .then((results) =>\n results.map((it) => ({\n ...it,\n similarity: 1 - it.distance,\n distance: it.distance,\n })),\n );\n}\n", "import { WriteAheadLog, open } from '@zukhruf/sqlite-open';\nimport * as sqliteVec from 'sqlite-vec';\n\n// The store's statements use the vec0 module, so every connection that runs\n// them, the dev database of the declaration included, loads sqlite-vec. The\n// rows of vectors are large, so a new file gets 32 KiB pages; the write-ahead\n// log fixes the page size, so the journal sets it before it.\nexport function openWithVectors(path: string) {\n const db = open(path, {\n allowExtension: true,\n journal: new WriteAheadLog({ pageSize: 32768 }),\n });\n db.loadExtension(sqliteVec.getLoadablePath());\n return db;\n}\n", "import { Declaration } from '@zukhruf/sqlite-drift';\nimport { transaction } from '@zukhruf/sqlite-transaction';\nimport dedent from 'dedent';\nimport type { DatabaseSync } from 'node:sqlite';\n\nimport type {\n Chunk,\n Corpus,\n Embedder,\n SearchOptions,\n SearchResult,\n Store,\n} from '../store.ts';\nimport { SETUP, statements, vectors } from './ddl.sqlite.ts';\nimport { openWithVectors } from './open.ts';\n\nconst DEFAULT_TOP_N = 10;\n\nconst tables = new Declaration(statements, { rows: 'keep' });\n\nexport class SQLiteStore implements Store {\n readonly #db: DatabaseSync;\n constructor(db: DatabaseSync, dimension: number) {\n this.#db = db;\n this.#db.exec(SETUP);\n const chunks = new Declaration([vectors(dimension)], {\n rows: 'replace',\n devDatabase: () => openWithVectors(':memory:'),\n });\n transaction(\n this.#db,\n () => {\n const stale = chunks.drift(this.#db) === 'stale';\n chunks.apply(this.#db);\n // A replaced vec_chunks is empty, yet upsertDoc skips a document whose\n // cid is unchanged and similaritySearch skips a source that exists.\n // Deleting the sources cascades to their documents, so the next\n // ingest embeds every document again.\n if (stale) this.#db.exec('DELETE FROM sources');\n },\n { mode: 'immediate' },\n );\n tables.apply(this.#db);\n }\n\n #searchByDocument(inputs: {\n sourceId: string;\n documentId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.document_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(\n inputs.sourceId,\n inputs.documentId,\n inputs.embedding,\n inputs.k,\n );\n }\n #searchBySource(inputs: {\n sourceId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(inputs.sourceId, inputs.embedding, inputs.k);\n }\n async search(query: string, options: SearchOptions, embedder: Embedder) {\n const { embeddings } = await embedder([query]);\n if (!embeddings.length) {\n return [];\n }\n const vectorBlob = vectorToBlob(embeddings[0]);\n const topN = options.topN;\n\n if (options.documentId) {\n const rows = this.#searchByDocument({\n sourceId: options.sourceId,\n documentId: options.documentId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n\n const rows = this.#searchBySource({\n sourceId: options.sourceId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n #upsertSource(inputs: { sourceId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id) VALUES (?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t`);\n return stmt.run(inputs.sourceId);\n }\n #upsertSourceWithExpiry(inputs: { sourceId: string; expiresAt: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id, expires_at) VALUES (?, ?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET\n\t\t\t\t\tupdated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now'),\n\t\t\t\t\texpires_at=excluded.expires_at\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.expiresAt);\n }\n upsertDoc(inputs: {\n documentId: string;\n sourceId: string;\n cid: string;\n metadata?: Record<string, any>;\n }) {\n const stmt = this.#db.prepare(dedent`\n INSERT INTO documents (id, source_id, cid, metadata)\n VALUES (?, ?, ?, json(?))\n ON CONFLICT(id) DO UPDATE SET\n cid=excluded.cid,\n metadata=COALESCE(excluded.metadata, documents.metadata),\n updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n WHERE documents.cid != excluded.cid;\n `);\n return stmt.run(\n inputs.documentId,\n inputs.sourceId,\n inputs.cid,\n inputs.metadata ? JSON.stringify(inputs.metadata) : null,\n );\n }\n insertDoc(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO vec_chunks (source_id, document_id, content, embedding)\n\t\t\t\tVALUES (?, ?, ?, vec_normalize(vec_f32(?)))\n\t\t\t`);\n return (chunk: Chunk) => {\n stmt.run(\n inputs.sourceId,\n inputs.documentId,\n chunk.content,\n vectorToBlob(chunk.embedding),\n );\n };\n }\n delete(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tDELETE FROM vec_chunks WHERE source_id = ? AND document_id = ?\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.documentId);\n }\n sourceExists(sourceId: string) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources WHERE source_id = ? LIMIT 1\n\t\t\t`);\n const row = stmt.get(sourceId);\n return Boolean(row);\n }\n sourceExpired(sourceId: string, now: Date) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources\n\t\t\t\tWHERE source_id = ?\n\t\t\t\t\tAND expires_at IS NOT NULL\n\t\t\t\t\tAND expires_at <= ?\n\t\t\t\tLIMIT 1\n\t\t\t`);\n const row = stmt.get(sourceId, now.toISOString());\n return Boolean(row);\n }\n setSourceExpiry(sourceId: string, expiryDate: Date) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tUPDATE sources\n\t\t\t\tSET expires_at = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tWHERE source_id = ?\n\t\t\t`);\n stmt.run(expiryDate.toISOString(), sourceId);\n }\n async index(sourceId: string, corpus: Corpus, expiryDate?: Date) {\n if (expiryDate) {\n this.#upsertSourceWithExpiry({\n expiresAt: expiryDate.toISOString(),\n sourceId,\n });\n } else {\n this.#upsertSource({ sourceId });\n }\n\n const info = this.upsertDoc({\n documentId: corpus.id,\n sourceId,\n cid: corpus.cid,\n metadata: corpus.metadata,\n });\n const changed = info.changes > 0;\n\n if (!changed) {\n return;\n }\n\n const insert = this.insertDoc({ sourceId, documentId: corpus.id });\n // Delete previous rows once before inserting\n transaction(\n this.#db,\n () => {\n this.delete({ sourceId, documentId: corpus.id });\n },\n { mode: 'immediate' },\n );\n\n const batchSize = 32;\n let batch: Chunk[] = [];\n const flush = () => {\n if (!batch.length) return;\n transaction(\n this.#db,\n () => {\n for (let i = 0; i < batch.length; i++) {\n insert(batch[i]);\n }\n },\n { mode: 'immediate' },\n );\n batch = [];\n };\n\n for await (const chunk of corpus.chunker()) {\n batch.push(chunk);\n if (batch.length >= batchSize) flush();\n }\n // flush any remaining\n flush();\n }\n}\n\nexport function vectorToBlob(vector: number[] | Float32Array): Buffer {\n if (vector instanceof Float32Array) {\n // Copy into a fresh Buffer to avoid retaining references to a larger batch tensor buffer\n const copied = new Float32Array(vector.length);\n copied.set(vector);\n return Buffer.from(copied.buffer);\n }\n const floatArray = new Float32Array(vector);\n return Buffer.from(floatArray.buffer);\n}\n\nfunction toSearchResult(row: unknown): SearchResult {\n if (\n typeof row !== 'object' ||\n row === null ||\n !('content' in row) ||\n typeof row.content !== 'string' ||\n !('distance' in row) ||\n typeof row.distance !== 'number' ||\n !('document_id' in row) ||\n typeof row.document_id !== 'string' ||\n !('metadata' in row) ||\n (row.metadata !== null && typeof row.metadata !== 'string')\n ) {\n throw new Error('Unexpected vector search row');\n }\n return {\n content: row.content,\n distance: row.distance,\n document_id: row.document_id,\n metadata: parseMetadata(row.metadata),\n };\n}\n\nfunction parseMetadata(value: string | null): Record<string, unknown> | null {\n if (value === null) return null;\n try {\n const parsed: unknown = JSON.parse(value);\n return isRecord(parsed) ? parsed : null;\n } catch {\n return null;\n }\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n", "// Embedding store schema for SQLite. SQLite stores the text of each CREATE and\n// @zukhruf/sqlite-drift compares it, so the text after each object name must\n// stay byte-identical to what earlier releases wrote, inline comments included.\n\n// Session-level PRAGMAs, run on each connection before the schema is applied.\nexport const SETUP = `\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n`;\n\nexport function vectors(dimension: number) {\n return `CREATE VIRTUAL TABLE vec_chunks USING vec0(\n source_id TEXT PARTITION KEY,\n document_id TEXT,\n embedding FLOAT[${dimension}] DISTANCE_METRIC=cosine,\n +content TEXT -- auxiliary payload (not filterable) for reconstruction\n)`;\n}\n\nexport const statements = [\n `CREATE TABLE sources (\n source_id TEXT PRIMARY KEY,\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n expires_at TEXT\n) STRICT`,\n `CREATE TABLE documents (\n id TEXT PRIMARY KEY,\n source_id TEXT NOT NULL,\n cid TEXT NOT NULL,\n metadata TEXT, -- JSON blob with arbitrary document metadata\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT`,\n // Indexes to accelerate lookups / maintenance\n `CREATE INDEX idx_documents_source ON documents(source_id)`,\n `CREATE INDEX idx_documents_source_updated ON documents(source_id, updated_at)`,\n `CREATE INDEX idx_documents_cid ON documents(cid)`,\n `CREATE INDEX idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL`,\n // emulate cascade for the virtual table\n `CREATE TRIGGER trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n DELETE FROM vec_chunks\n WHERE source_id = OLD.source_id\n AND document_id = OLD.id;\nEND`,\n];\n", "import { openWithVectors } from './open.ts';\nimport { SQLiteStore } from './sqlite.ts';\n\nexport function nodeSQLite(dbName: string, dimension: number) {\n return new SQLiteStore(openWithVectors(dbName), dimension);\n}\n"],
|
|
5
|
+
"mappings": ";AA6BO,SAAS,mBAAmB,QAAgB;AACjD,QAAM,OAAO,OAAO;AACpB,QAAM,CAAC,WAAW,GAAG,IAAI,IAAI,MAAM,QAAQ,IAAI,IAAI,OAAO,CAAC;AAC3D,QAAM,aAAa,KAAK,GAAG,EAAE;AAC7B,MAAI,eAAe,QAAW;AAC5B,UAAM,IAAI,MAAM,2BAA2B,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,EACnE;AACA,QAAM,cAAc,YAAY;AAChC,MAAI,OAAO,KAAK,WAAW,aAAa;AACtC,UAAM,IAAI;AAAA,MACR,6BAA6B,OAAO,KAAK,MAAM,cAAc,WAAW,WAAW,SAAS,YAAY,UAAU;AAAA,IACpH;AAAA,EACF;AACA,QAAM,OAAO,OAAO;AACpB,MAAI,EAAE,gBAAgB,eAAe;AACnC,UAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AAAA,EACnE;AAEA,QAAM,aAA6B,CAAC;AACpC,QAAM,aAAa;AACnB,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,IAAI;AAClB,eAAW,KAAK,KAAK,SAAS,OAAO,QAAQ,UAAU,CAAC;AAAA,EAC1D;AACA,SAAO,EAAE,YAAY,WAAW;AAClC;;;AC9CO,IAAM,kBAAkB;AAAA,EAC7B,eAAe;AAAA,EACf,WAAW;AAAA,EACX,cAAc;AAAA,EACd,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,WAAW;AACb;AAIA,IAAM,oBAAoD;AAAA,EACxD,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,SAAS,GAAG;AAAA,EAC7B,CAAC,gBAAgB,YAAY,GAAG;AAAA,EAChC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,SAAS,GAAG;AAC/B;AAaO,SAAS,UAAU,UAA4B,CAAC,GAAa;AAClE,QAAM;AAAA,IACJ,OAAO,UAAU,gBAAgB;AAAA,IACjC,YAAY;AAAA,IACZ;AAAA,EACF,IAAI;AAEJ,MAAI,CAAC,OAAO,UAAU,SAAS,KAAK,aAAa,GAAG;AAClD,UAAM,IAAI,WAAW,sCAAsC;AAAA,EAC7D;AAEA,MAAI,eAA0D;AAC9D,QAAM,WAAW,MAAM;AACrB,QAAI,CAAC,cAAc;AACjB,qBAAe,OAAO,2BAA2B,EAAE;AAAA,QAAK,CAAC,EAAE,SAAS,MAClE,SAAS,sBAAsB,kBAAkB,OAAO,GAAG;AAAA,UACzD,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,SAAO,OAAO,cAAwB;AACpC,QAAI,UAAU,WAAW,GAAG;AAC1B,aAAO,EAAE,YAAY,CAAC,GAAG,YAAY,EAAE;AAAA,IACzC;AAEA,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,aAA6B,CAAC;AACpC,QAAI,aAAa;AAEjB,aAAS,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,WAAW;AAChE,YAAM,SAAS,MAAM,MAAM,UAAU,MAAM,OAAO,QAAQ,SAAS,GAAG;AAAA,QACpE,SAAS;AAAA,QACT,WAAW;AAAA,MACb,CAAC;AACD,YAAM,QAAQ,mBAAmB,MAAM;AACvC,qBAAe,MAAM;AACrB,iBAAW,KAAK,GAAG,MAAM,UAAU;AAAA,IACrC;AAEA,WAAO,EAAE,YAAY,WAAW;AAAA,EAClC;AACF;;;ACrFA;AAAA,EACE;AAAA,EACA;AAAA,OACK;;;ACHP,SAAS,YAAY;AAEd,SAAS,IAAI,SAAiB;AACnC,SAAO,KAAK,UAAU,OAAO;AAC/B;;;ADaA,eAAsB,OACpB,QACA,UACA;AACA,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,WAAW,OAAO;AACxB,QAAM,WAAW,OAAO,UAAU,QAAQ;AAC1C,QAAM,aAAa,OAAO,OAAO,SAAS;AAE1C,mBAAiB,MAAM,UAAU;AAC/B,eAAW,GAAG,EAAE;AAChB,UAAM,UAAU,MAAM,GAAG,QAAQ;AACjC,QAAI,CAAC,QAAQ,KAAK,GAAG;AAEnB;AAAA,IACF;AACA,UAAM,OAAO,MAAM;AAAA,MACjB,OAAO,UAAU;AAAA,MACjB;AAAA,QACE,IAAI,GAAG;AAAA,QACP,KAAK,IAAI,OAAO;AAAA,QAChB,UAAU,GAAG;AAAA,QACb,SAAS,mBAAmB;AAE1B,gBAAM,SAAS,MAAM,SAAS,GAAG,IAAI,OAAO;AAC5C,gBAAM,YAAY;AAClB,mBAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,WAAW;AACjD,kBAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,SAAS;AAC3C,kBAAM,EAAE,WAAW,IAAI,MAAM,SAAS,KAAK;AAC3C,qBAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,oBAAM;AAAA,gBACJ,SAAS,MAAM,CAAC;AAAA,gBAChB,WAAW,WAAW,CAAC;AAAA,cACzB;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,OAAO,WAAsB;AACpC,MAAI,UAAU,eAAe,UAAW,QAAO;AAC/C,MAAI,UAAU,iBAAiB,OAAW,QAAO;AACjD,SAAO,IAAI,KAAK,UAAU,MAAM,KAAK,IAAI,IAAI,UAAU,YAAY;AACrE;AAEA,SAAS,MAAM,IAAY,SAAiB;AAC1C,QAAM,WAAW,IAAI,qBAAqB;AAC1C,SAAO,SAAS,UAAU,OAAO;AACnC;AAeA,SAAS,kBAAkB,OAAe;AACxC,SAAO,MAAM,QAAQ,SAAS,IAAI;AACpC;AAEA,SAAS,iBACP,iBACA,QAC6B;AAC7B,MAAI,CAAC,OAAO,QAAQ;AAClB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,kBAAkB,eAAe;AAC3D,QAAM,YAAyC,CAAC;AAChD,MAAI,eAAe;AAEnB,aAAW,SAAS,QAAQ;AAC1B,UAAM,kBAAkB,kBAAkB,KAAK;AAC/C,UAAM,eAAe,gBAAgB,KAAK;AAE1C,UAAM,OAAO,CAAC,QAAgB,cAC5B,SAAS,kBAAkB,QAAQ,QAAQ,SAAS,IAAI;AAE1D,QAAI,aAAa,KAAK,iBAAiB,YAAY;AACnD,QAAI,aAAa;AAEjB,QAAI,eAAe,MAAM,cAAc;AACrC,mBAAa,KAAK,cAAc,YAAY;AAC5C,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,IAAI;AACrB,mBAAa,KAAK,iBAAiB,CAAC;AACpC,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,MAAM,cAAc;AACrC,mBAAa,KAAK,cAAc,CAAC;AACjC,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,IAAI;AACrB,gBAAU,KAAK,IAAI;AACnB;AAAA,IACF;AAEA,UAAM,SAAS,kBAAkB,MAAM,GAAG,UAAU;AACpD,UAAM,cAAc,OAAO,MAAM,IAAI;AACrC,UAAM,YAAY,YAAY;AAC9B,UAAM,cAAc,OAAO,SAAS,OAAO,YAAY,IAAI;AAE3D,UAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,UAAM,UAAU,YAAY,MAAM,SAAS;AAC3C,UAAM,YACJ,MAAM,WAAW,IACb,cAAc,MAAM,CAAC,EAAE,SACvB,WAAW,SAAS,WAAW,YAAY,IAAI;AAErD,cAAU,KAAK,EAAE,WAAW,aAAa,SAAS,UAAU,CAAC;AAC7D,mBAAe,aAAa,WAAW;AAAA,EACzC;AAEA,SAAO;AACT;AAEA,SAAS,yBACP,iBACA,QAC0B;AAC1B,QAAM,YAAY,iBAAiB,iBAAiB,MAAM;AAC1D,SAAO,OAAO,IAAI,CAAC,SAAS,WAAW;AAAA,IACrC;AAAA,IACA;AAAA,IACA,UAAU,UAAU,KAAK,KAAK;AAAA,EAChC,EAAE;AACJ;AAEA,eAAsB,6BACpB,IACA,SACmC;AACnC,QAAM,WAAW,+BAA+B,aAAa,MAAM;AAAA,IACjE,WAAW;AAAA,IACX,cAAc;AAAA,EAChB,CAAC;AACD,QAAM,OAAO,MAAM,SAAS,gBAAgB,CAAC,OAAO,CAAC;AACrD,QAAM,SAAS,KAAK,IAAI,CAAC,MAAM,EAAE,WAAW;AAC5C,SAAO,yBAAyB,SAAS,MAAM;AACjD;AAEA,eAAsB,gBAAgB,IAAY,SAAiB;AACjE,QAAM,SAAS,MAAM,6BAA6B,IAAI,OAAO;AAC7D,SAAO,OAAO,IAAI,CAAC,UAAU,MAAM,OAAO;AAC5C;;;AEjLA,OAA0B;AAI1B,eAAsB,iBACpB,OACA,QACA;AAEA,QAAM,EAAE,UAAU,IAAI;AACtB,MAAI,eAAe;AAEnB,MAAI,UAAU,eAAe,SAAS;AACpC,QAAI,MAAM,OAAO,MAAM,aAAa,OAAO,UAAU,QAAQ,GAAG;AAC9D,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF,WAAW,UAAU,eAAe,WAAW;AAC7C,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,IACnB;AACA,QACE,gBACA,CAAE,MAAM,OAAO,MAAM;AAAA,MACnB,OAAO,UAAU;AAAA,MACjB,IAAI,KAAK,UAAU,MAAM,KAAK,IAAI,CAAC;AAAA,IACrC,GACA;AACA,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAEA,MAAI,cAAc;AAChB,UAAM,OAAO,MAAM;AAAA,EACrB;AACA,SAAO,OAAO,MACX;AAAA,IACC;AAAA,IACA,EAAE,UAAU,OAAO,UAAU,UAAU,MAAM,GAAG;AAAA,IAChD,OAAO;AAAA,EACT,EACC;AAAA,IAAK,CAAC,YACL,QAAQ,IAAI,CAAC,QAAQ;AAAA,MACnB,GAAG;AAAA,MACH,YAAY,IAAI,GAAG;AAAA,MACnB,UAAU,GAAG;AAAA,IACf,EAAE;AAAA,EACJ;AACJ;;;ACrDA,SAAS,eAAe,YAAY;AACpC,YAAY,eAAe;AAMpB,SAAS,gBAAgB,MAAc;AAC5C,QAAM,KAAK,KAAK,MAAM;AAAA,IACpB,gBAAgB;AAAA,IAChB,SAAS,IAAI,cAAc,EAAE,UAAU,MAAM,CAAC;AAAA,EAChD,CAAC;AACD,KAAG,cAAwB,0BAAgB,CAAC;AAC5C,SAAO;AACT;;;ACdA,SAAS,mBAAmB;AAC5B,SAAS,mBAAmB;AAC5B,OAAO,YAAY;;;ACGZ,IAAM,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AASd,SAAS,QAAQ,WAAmB;AACzC,SAAO;AAAA;AAAA;AAAA,oBAGW,SAAS;AAAA;AAAA;AAG7B;AAEO,IAAM,aAAa;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA;AAAA,EAKA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOF;;;ADlCA,IAAM,gBAAgB;AAEtB,IAAM,SAAS,IAAI,YAAY,YAAY,EAAE,MAAM,OAAO,CAAC;AAEpD,IAAM,cAAN,MAAmC;AAAA,EAC/B;AAAA,EACT,YAAY,IAAkB,WAAmB;AAC/C,SAAK,MAAM;AACX,SAAK,IAAI,KAAK,KAAK;AACnB,UAAM,SAAS,IAAI,YAAY,CAAC,QAAQ,SAAS,CAAC,GAAG;AAAA,MACnD,MAAM;AAAA,MACN,aAAa,MAAM,gBAAgB,UAAU;AAAA,IAC/C,CAAC;AACD;AAAA,MACE,KAAK;AAAA,MACL,MAAM;AACJ,cAAM,QAAQ,OAAO,MAAM,KAAK,GAAG,MAAM;AACzC,eAAO,MAAM,KAAK,GAAG;AAKrB,YAAI,MAAO,MAAK,IAAI,KAAK,qBAAqB;AAAA,MAChD;AAAA,MACA,EAAE,MAAM,YAAY;AAAA,IACtB;AACA,WAAO,MAAM,KAAK,GAAG;AAAA,EACvB;AAAA,EAEA,kBAAkB,QAKf;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAS9B;AACA,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,gBAAgB,QAIb;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQ9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,WAAW,OAAO,CAAC;AAAA,EAC7D;AAAA,EACA,MAAM,OAAO,OAAe,SAAwB,UAAoB;AACtE,UAAM,EAAE,WAAW,IAAI,MAAM,SAAS,CAAC,KAAK,CAAC;AAC7C,QAAI,CAAC,WAAW,QAAQ;AACtB,aAAO,CAAC;AAAA,IACV;AACA,UAAM,aAAa,aAAa,WAAW,CAAC,CAAC;AAC7C,UAAM,OAAO,QAAQ;AAErB,QAAI,QAAQ,YAAY;AACtB,YAAMA,QAAO,KAAK,kBAAkB;AAAA,QAClC,UAAU,QAAQ;AAAA,QAClB,YAAY,QAAQ;AAAA,QACpB,WAAW;AAAA,QACX,GAAG,QAAQ;AAAA,MACb,CAAC;AACD,aAAOA,MAAK,IAAI,cAAc;AAAA,IAChC;AAEA,UAAM,OAAO,KAAK,gBAAgB;AAAA,MAChC,UAAU,QAAQ;AAAA,MAClB,WAAW;AAAA,MACX,GAAG,QAAQ;AAAA,IACb,CAAC;AACD,WAAO,KAAK,IAAI,cAAc;AAAA,EAChC;AAAA,EACA,cAAc,QAA8B;AAC1C,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,KAAK,IAAI,OAAO,QAAQ;AAAA,EACjC;AAAA,EACA,wBAAwB,QAAiD;AACvE,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAK9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,SAAS;AAAA,EACnD;AAAA,EACA,UAAU,QAKP;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,OAQ3B;AACH,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO,WAAW,KAAK,UAAU,OAAO,QAAQ,IAAI;AAAA,IACtD;AAAA,EACF;AAAA,EACA,UAAU,QAAkD;AAC1D,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,CAAC,UAAiB;AACvB,WAAK;AAAA,QACH,OAAO;AAAA,QACP,OAAO;AAAA,QACP,MAAM;AAAA,QACN,aAAa,MAAM,SAAS;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAAA,EACA,OAAO,QAAkD;AACvD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,UAAU;AAAA,EACpD;AAAA,EACA,aAAa,UAAkB;AAC7B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,cAAc,UAAkB,KAAW;AACzC,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAM9B;AACA,UAAM,MAAM,KAAK,IAAI,UAAU,IAAI,YAAY,CAAC;AAChD,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,gBAAgB,UAAkB,YAAkB;AAClD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA,IAI9B;AACA,SAAK,IAAI,WAAW,YAAY,GAAG,QAAQ;AAAA,EAC7C;AAAA,EACA,MAAM,MAAM,UAAkB,QAAgB,YAAmB;AAC/D,QAAI,YAAY;AACd,WAAK,wBAAwB;AAAA,QAC3B,WAAW,WAAW,YAAY;AAAA,QAClC;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,WAAK,cAAc,EAAE,SAAS,CAAC;AAAA,IACjC;AAEA,UAAM,OAAO,KAAK,UAAU;AAAA,MAC1B,YAAY,OAAO;AAAA,MACnB;AAAA,MACA,KAAK,OAAO;AAAA,MACZ,UAAU,OAAO;AAAA,IACnB,CAAC;AACD,UAAM,UAAU,KAAK,UAAU;AAE/B,QAAI,CAAC,SAAS;AACZ;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,UAAU,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAEjE;AAAA,MACE,KAAK;AAAA,MACL,MAAM;AACJ,aAAK,OAAO,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAAA,MACjD;AAAA,MACA,EAAE,MAAM,YAAY;AAAA,IACtB;AAEA,UAAM,YAAY;AAClB,QAAI,QAAiB,CAAC;AACtB,UAAM,QAAQ,MAAM;AAClB,UAAI,CAAC,MAAM,OAAQ;AACnB;AAAA,QACE,KAAK;AAAA,QACL,MAAM;AACJ,mBAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,mBAAO,MAAM,CAAC,CAAC;AAAA,UACjB;AAAA,QACF;AAAA,QACA,EAAE,MAAM,YAAY;AAAA,MACtB;AACA,cAAQ,CAAC;AAAA,IACX;AAEA,qBAAiB,SAAS,OAAO,QAAQ,GAAG;AAC1C,YAAM,KAAK,KAAK;AAChB,UAAI,MAAM,UAAU,UAAW,OAAM;AAAA,IACvC;AAEA,UAAM;AAAA,EACR;AACF;AAEO,SAAS,aAAa,QAAyC;AACpE,MAAI,kBAAkB,cAAc;AAElC,UAAM,SAAS,IAAI,aAAa,OAAO,MAAM;AAC7C,WAAO,IAAI,MAAM;AACjB,WAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAClC;AACA,QAAM,aAAa,IAAI,aAAa,MAAM;AAC1C,SAAO,OAAO,KAAK,WAAW,MAAM;AACtC;AAEA,SAAS,eAAe,KAA4B;AAClD,MACE,OAAO,QAAQ,YACf,QAAQ,QACR,EAAE,aAAa,QACf,OAAO,IAAI,YAAY,YACvB,EAAE,cAAc,QAChB,OAAO,IAAI,aAAa,YACxB,EAAE,iBAAiB,QACnB,OAAO,IAAI,gBAAgB,YAC3B,EAAE,cAAc,QACf,IAAI,aAAa,QAAQ,OAAO,IAAI,aAAa,UAClD;AACA,UAAM,IAAI,MAAM,8BAA8B;AAAA,EAChD;AACA,SAAO;AAAA,IACL,SAAS,IAAI;AAAA,IACb,UAAU,IAAI;AAAA,IACd,aAAa,IAAI;AAAA,IACjB,UAAU,cAAc,IAAI,QAAQ;AAAA,EACtC;AACF;AAEA,SAAS,cAAc,OAAsD;AAC3E,MAAI,UAAU,KAAM,QAAO;AAC3B,MAAI;AACF,UAAM,SAAkB,KAAK,MAAM,KAAK;AACxC,WAAO,SAAS,MAAM,IAAI,SAAS;AAAA,EACrC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;;;AErSO,SAAS,WAAW,QAAgB,WAAmB;AAC5D,SAAO,IAAI,YAAY,gBAAgB,MAAM,GAAG,SAAS;AAC3D;",
|
|
6
6
|
"names": ["rows"]
|
|
7
7
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
interface ConnectorSource {
|
|
2
2
|
/**
|
|
3
3
|
* List of corpuses
|
|
4
4
|
*/
|
|
@@ -9,17 +9,31 @@ export type Connector = {
|
|
|
9
9
|
}, void, unknown>;
|
|
10
10
|
/** Unique identifier for the logical source (group of documents) used in the embedding store. */
|
|
11
11
|
sourceId: string;
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Controls ingestion behavior:
|
|
15
|
+
* - 'never': perform ingestion only if the source does NOT yet exist; once created, never re-ingest
|
|
16
|
+
* - 'contentChanged': (default) run ingestion; underlying pipeline will skip unchanged documents
|
|
17
|
+
* - 'expired': only ingest if source doesn't exist OR is expired
|
|
18
|
+
*
|
|
19
|
+
* Only the 'expired' mode reads the time, so only it has an expiry and a clock.
|
|
20
|
+
*/
|
|
21
|
+
export type IngestPolicy = {
|
|
22
|
+
ingestWhen?: 'never' | 'contentChanged';
|
|
23
|
+
} | {
|
|
24
|
+
ingestWhen: 'expired';
|
|
19
25
|
/**
|
|
20
26
|
* Optional expiry duration in milliseconds from now.
|
|
21
27
|
* When set, the source will be considered expired after this duration.
|
|
22
28
|
*/
|
|
23
29
|
expiresAfter?: number;
|
|
30
|
+
/** An entry gives `globalThis`; a test gives `createClock()`. */
|
|
31
|
+
clock: {
|
|
32
|
+
Date: {
|
|
33
|
+
now(): number;
|
|
34
|
+
};
|
|
35
|
+
};
|
|
24
36
|
};
|
|
37
|
+
export type Connector = ConnectorSource & IngestPolicy;
|
|
38
|
+
export {};
|
|
25
39
|
//# sourceMappingURL=connector.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connector.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/connector.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"connector.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/connector.ts"],"names":[],"mappings":"AAAA,UAAU,eAAe;IACvB;;OAEG;IACH,OAAO,EAAE,MAAM,cAAc,CAC3B;QACE,EAAE,EAAE,MAAM,CAAC;QACX,OAAO,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;KAChD,EACD,IAAI,EACJ,OAAO,CACR,CAAC;IACF,iGAAiG;IACjG,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,YAAY,GACpB;IAAE,UAAU,CAAC,EAAE,OAAO,GAAG,gBAAgB,CAAA;CAAE,GAC3C;IACE,UAAU,EAAE,SAAS,CAAC;IACtB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iEAAiE;IACjE,KAAK,EAAE;QAAE,IAAI,EAAE;YAAE,GAAG,IAAI,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;CACpC,CAAC;AAEN,MAAM,MAAM,SAAS,GAAG,eAAe,GAAG,YAAY,CAAC"}
|
|
@@ -262,17 +262,14 @@ async function getFilteredFiles(cwd, pattern) {
|
|
|
262
262
|
}
|
|
263
263
|
return filteredFiles.sort();
|
|
264
264
|
}
|
|
265
|
-
function local(pattern, options) {
|
|
265
|
+
function local(pattern, options = {}) {
|
|
266
|
+
const { cwd, ...policy } = options;
|
|
266
267
|
const sourceId = `glob:${pattern}`;
|
|
267
268
|
return {
|
|
269
|
+
...policy,
|
|
268
270
|
sourceId,
|
|
269
|
-
ingestWhen: options?.ingestWhen,
|
|
270
|
-
expiresAfter: options?.expiresAfter,
|
|
271
271
|
sources: async function* () {
|
|
272
|
-
const paths = await getFilteredFiles(
|
|
273
|
-
options?.cwd ?? process.cwd(),
|
|
274
|
-
pattern
|
|
275
|
-
);
|
|
272
|
+
const paths = await getFilteredFiles(cwd ?? process.cwd(), pattern);
|
|
276
273
|
for (const filePath of paths) {
|
|
277
274
|
yield {
|
|
278
275
|
id: filePath,
|
|
@@ -378,7 +375,8 @@ async function parseRSSFeed(feedUrl) {
|
|
|
378
375
|
} catch (error) {
|
|
379
376
|
console.error(`Failed to parse RSS feed ${feedUrl}:`, error);
|
|
380
377
|
throw new Error(
|
|
381
|
-
`RSS parsing failed: ${error
|
|
378
|
+
`RSS parsing failed: ${Error.isError(error) ? error.message : "Unknown error"}`,
|
|
379
|
+
{ cause: error }
|
|
382
380
|
);
|
|
383
381
|
}
|
|
384
382
|
}
|
|
@@ -408,7 +406,7 @@ ${article.content}` : article.content;
|
|
|
408
406
|
} catch (error) {
|
|
409
407
|
console.warn(
|
|
410
408
|
`Failed to fetch article content from ${url}:`,
|
|
411
|
-
error
|
|
409
|
+
Error.isError(error) ? error.message : error
|
|
412
410
|
);
|
|
413
411
|
return "";
|
|
414
412
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/lib/connectors/github.ts", "../../../src/lib/connectors/linear.ts", "../../../src/lib/connectors/local.ts", "../../../src/lib/connectors/pdf.ts", "../../../src/lib/connectors/rss.ts", "../../../src/lib/connectors/repo.ts"],
|
|
4
|
-
"sourcesContent": ["import { execa } from 'execa';\n\nimport type { Connector } from './connector.ts';\n\ninterface ReleaseFetchOptions {\n /** Stop fetching when this tag is encountered (latest-first order). */\n untilTag?: string;\n /** Include the matching untilTag release (default true). */\n inclusive?: boolean;\n /** Include draft releases (default false). */\n includeDrafts?: boolean;\n /** Include prereleases (default false). */\n includePrerelease?: boolean;\n}\n\ninterface Release {\n tag_name: string;\n name: string | null;\n body: string | null;\n updated_at: string;\n published_at: string;\n html_url: string;\n draft: boolean;\n prerelease: boolean;\n}\n\nfunction isRelease(value: unknown): value is Release {\n return (\n typeof value === 'object' &&\n value !== null &&\n 'tag_name' in value &&\n typeof value.tag_name === 'string' &&\n 'name' in value &&\n (value.name === null || typeof value.name === 'string') &&\n 'body' in value &&\n (value.body === null || typeof value.body === 'string') &&\n 'updated_at' in value &&\n typeof value.updated_at === 'string' &&\n 'published_at' in value &&\n typeof value.published_at === 'string' &&\n 'html_url' in value &&\n typeof value.html_url === 'string' &&\n 'draft' in value &&\n typeof value.draft === 'boolean' &&\n 'prerelease' in value &&\n typeof value.prerelease === 'boolean'\n );\n}\n\nfunction fs(path: string) {\n const [owner, repo, ...filePath] = path.split('/');\n return {\n readFile: async () => {\n const url = `https://api.github.com/repos/${owner}/${repo}/contents/${filePath.join('/')}`;\n const res = await fetch(url);\n const data: unknown = await res.json();\n if (\n typeof data !== 'object' ||\n data === null ||\n !('content' in data) ||\n typeof data.content !== 'string'\n ) {\n throw new Error(`GitHub returned no file content for ${path}`);\n }\n return atob(data.content);\n },\n release: async (repo: string, opts: ReleaseFetchOptions = {}) => {\n const [owner, repoName] = repo.split('/');\n const perPage = 100; // always fetch max per page for efficiency\n const maxPages = 10; // internal safety cap, not user configurable\n const releases: Release[] = [];\n let stop = false;\n\n for (let page = 1; page <= maxPages && !stop; page++) {\n const url = `https://api.github.com/repos/${owner}/${repoName}/releases?per_page=${perPage}&page=${page}`;\n const res = await fetch(url, {\n headers: { Accept: 'application/vnd.github+json' },\n });\n if (!res.ok) break;\n const pageData: unknown = await res.json();\n if (!Array.isArray(pageData) || !pageData.every(isRelease)) {\n throw new Error(`Unexpected GitHub releases response for ${repo}`);\n }\n if (!pageData.length) break;\n for (const rel of pageData) {\n if (!opts.includeDrafts && rel.draft) continue;\n if (!opts.includePrerelease && rel.prerelease) continue;\n\n releases.push(rel);\n if (opts.untilTag && rel.tag_name === opts.untilTag) {\n if (opts.inclusive === false) {\n releases.pop();\n }\n stop = true;\n break;\n }\n }\n }\n return releases;\n },\n };\n}\n\nexport const github = {\n file(filePath: string): Connector {\n const sourceId = `github:file:${filePath}`;\n const documentId = filePath;\n return {\n sourceId,\n sources: async function* () {\n const { readFile } = fs(filePath);\n yield {\n id: documentId,\n content: () => readFile(),\n };\n },\n };\n },\n /**\n * Create a connector over GitHub Releases with optional pagination & early-stop.\n * @param repo owner/repo string\n * @param options Release filtering & pagination controls\n */\n release: (repo: string, options: ReleaseFetchOptions = {}): Connector => {\n const sourceId = `github:releases:${repo}`;\n return {\n sourceId,\n sources: async function* () {\n const releases = await fs(repo).release(repo, options);\n for (const rel of releases) {\n yield {\n id: `${repo}:release:${rel.tag_name}`,\n content: async () => {\n const name = rel.name || rel.tag_name;\n const body = rel.body || '';\n return `Release: ${name}\\nTag: ${rel.tag_name}\\nPublished at: ${rel.published_at}\\nUpdated at: ${rel.updated_at}\\nURL: ${rel.html_url}\\nDraft: ${rel.draft}\\nPrerelease: ${rel.prerelease}\\n\\n${body}`;\n },\n };\n }\n },\n };\n },\n /**\n * Create a connector over a GitHub repository file tree using gitingest to produce a markdown digest.\n * @param repoUrl full https://github.com/OWNER/REPO or .../tree/BRANCH/subdir URL\n */\n repo: (\n repoUrl: string,\n opts: {\n includes: string[];\n excludes?: string[];\n branch?: string;\n includeGitignored?: boolean;\n includeSubmodules?: boolean;\n githubToken?: string;\n ingestWhen?: 'never' | 'contentChanged';\n },\n ): Connector => {\n const sourceId = `github:repo:${repoUrl}`;\n const {\n excludes = [\n '**/node_modules/**',\n '**/dist/**',\n '**/coverage/**',\n '**/*.test.ts',\n '**/*.test.tsx',\n '**/dist/**',\n '**/.git/**',\n '**/.github/**',\n '**/.vscode/**',\n '**/.idea/**',\n '**/build/**',\n '**/out/**',\n '**/vendor/**',\n '**/__tests__/**',\n '**/*.d.ts',\n ],\n branch,\n includeGitignored = false,\n includeSubmodules = false,\n githubToken,\n ingestWhen,\n includes,\n } = opts;\n\n async function gitingestDigest(): Promise<string> {\n const args = [\n 'gitingest',\n ...(branch ? ['-b', branch] : []),\n ...includes.flatMap((p) => ['-i', p]),\n ...excludes.flatMap((p) => ['-e', p]),\n ...(includeGitignored ? ['--include-gitignored'] : []),\n ...(includeSubmodules ? ['--include-submodules'] : []),\n repoUrl,\n '-o',\n '-',\n ];\n const { stdout } = await execa('uvx', args, {\n env: githubToken\n ? { ...process.env, GITHUB_TOKEN: githubToken }\n : process.env,\n stdout: 'pipe',\n stderr: 'inherit',\n });\n return stdout;\n }\n\n return {\n sourceId,\n ingestWhen,\n sources: async function* () {\n const content = await gitingestDigest();\n yield {\n id: 'digest.md',\n content: async () => content,\n };\n },\n };\n },\n};\n", "import { LinearClient } from '@linear/sdk';\n\nexport function linear(apiKey: string) {\n const linearClient = new LinearClient({ apiKey });\n const sourceId = 'linear:workspace';\n return {\n sourceId,\n instructions:\n 'You answer questions about Linear issues assigned to the user.',\n sources: async () => {\n const user = await linearClient.viewer;\n const issues = await user.assignedIssues();\n return Promise.all(\n issues.nodes.map(async (it) => {\n const state = await it.state;\n return {\n id: it.id,\n content: async () =>\n `Issue: ${it.title}\\nDescription: ${it.description || 'No description'}\\nStatus: ${state?.name || 'Unknown'}\\nID: ${it.id}`,\n };\n }),\n );\n },\n };\n}\n", "import fg from 'fast-glob';\nimport ignore from 'ignore';\nimport { readFile, stat } from 'node:fs/promises';\nimport { dirname, join, relative, resolve } from 'node:path';\n\nimport type { Connector } from './connector.ts';\n\n/**\n * Cache for gitignore patterns to avoid repeated file reads\n */\nconst gitignoreCache = new Map<string, ReturnType<typeof ignore>>();\n\n/**\n * Collect all .gitignore patterns from root directory up to target directory\n */\nasync function collectGitignorePatterns(\n targetPath: string,\n): Promise<ReturnType<typeof ignore>> {\n const absolutePath = resolve(targetPath);\n\n if (gitignoreCache.has(absolutePath)) {\n return gitignoreCache.get(absolutePath)!;\n }\n\n const ig = ignore();\n\n // Find all .gitignore files from root to target directory\n const pathSegments = absolutePath.split('/');\n const gitignoreFiles: string[] = [];\n\n // Check each level from root to target\n for (let i = 1; i <= pathSegments.length; i++) {\n const currentPath = pathSegments.slice(0, i).join('/');\n const gitignorePath = join(currentPath, '.gitignore');\n\n try {\n await stat(gitignorePath);\n gitignoreFiles.push(gitignorePath);\n } catch {\n // .gitignore doesn't exist at this level, continue\n }\n }\n\n // Read and add patterns from all .gitignore files\n for (const gitignoreFile of gitignoreFiles) {\n try {\n const content = await readFile(gitignoreFile, 'utf8');\n const patterns = content\n .split('\\n')\n .map((line) => line.trim())\n .filter((line) => line && !line.startsWith('#'));\n\n ig.add(patterns);\n } catch {\n // Skip if can't read gitignore file\n }\n }\n\n gitignoreCache.set(absolutePath, ig);\n return ig;\n}\n\n/**\n * Get all files matching the pattern while respecting .gitignore files\n */\nasync function getFilteredFiles(\n cwd: string,\n pattern: string,\n): Promise<string[]> {\n // Get all files matching the pattern without gitignore filtering first\n const allFiles = await fg(pattern, {\n dot: false,\n absolute: true,\n onlyFiles: true,\n cwd: cwd,\n followSymbolicLinks: false,\n ignore: [\n '**/node_modules/**',\n '**/.git/**',\n '**/.DS_Store',\n '**/Thumbs.db',\n '**/*.tmp',\n '**/*.temp',\n '**/coverage/**',\n '**/dist/**',\n '**/build/**',\n ],\n });\n\n if (allFiles.length === 0) {\n return [];\n }\n\n // Group files by their directory to optimize gitignore pattern collection\n const filesByDir = new Map<string, string[]>();\n\n for (const file of allFiles) {\n const dir = dirname(file);\n if (!filesByDir.has(dir)) {\n filesByDir.set(dir, []);\n }\n filesByDir.get(dir)!.push(file);\n }\n\n const filteredFiles: string[] = [];\n\n // Process each directory and filter files\n for (const [dir, files] of filesByDir) {\n const ignoreFilter = await collectGitignorePatterns(dir);\n\n for (const file of files) {\n if (!ignoreFilter.ignores(relative(cwd, file))) {\n filteredFiles.push(file);\n }\n }\n }\n\n return filteredFiles.sort();\n}\n\nexport function local(\n pattern: string,\n options?: {\n ingestWhen?: 'never' | 'contentChanged' | 'expired';\n expiresAfter?: number;\n cwd?: string;\n },\n): Connector {\n const sourceId = `glob:${pattern}`;\n return {\n sourceId,\n ingestWhen: options?.ingestWhen,\n expiresAfter: options?.expiresAfter,\n sources: async function* () {\n const paths = await getFilteredFiles(\n options?.cwd ?? process.cwd(),\n pattern,\n );\n for (const filePath of paths) {\n yield {\n id: filePath,\n content: () => readFile(filePath, 'utf8').catch(() => ''),\n };\n }\n },\n };\n}\n", "import { glob, readFile } from 'node:fs/promises';\nimport { extractText, getDocumentProxy } from 'unpdf';\n\nimport type { Connector } from './connector.ts';\n\nexport function pdf(pattern: string): Connector {\n const sourceId = `pdf:${pattern}`;\n\n return {\n sourceId,\n sources: async function* () {\n const paths = await Array.fromAsync(\n glob(pattern, { exclude: ['**/node_modules/**', '**/.git/**'] }),\n );\n for (const path of paths) {\n if (!path.toLowerCase().endsWith('.pdf')) continue;\n yield {\n id: path,\n content: async () => {\n const buffer = await readFile(path);\n const pdf = await getDocumentProxy(new Uint8Array(buffer));\n const { text } = await extractText(pdf, { mergePages: true });\n return text;\n },\n };\n }\n },\n };\n}\n\nexport function pdfFile(source: string): Connector {\n const isUrl = /^https?:\\/\\//.test(source);\n const sourceId = `pdf:${isUrl ? 'url' : 'file'}:${source}`;\n\n return {\n sourceId,\n sources: async function* () {\n yield {\n id: source,\n content: async () => {\n let buffer: Uint8Array;\n\n if (isUrl) {\n const response = await fetch(source);\n if (!response.ok) {\n throw new Error(\n `HTTP ${response.status}: ${response.statusText}`,\n );\n }\n buffer = new Uint8Array(await response.arrayBuffer());\n } else {\n const fileBuffer = await readFile(source);\n buffer = new Uint8Array(fileBuffer.buffer);\n }\n\n const pdf = await getDocumentProxy(new Uint8Array(buffer));\n const { text } = await extractText(pdf, { mergePages: true });\n return text;\n },\n };\n },\n };\n}\n", "import { experimental_createWebFetchTool } from '@ai-sdk/tool-web-fetch';\nimport assert from 'node:assert/strict';\nimport Parser from 'rss-parser';\n\nconst webFetch = experimental_createWebFetchTool({\n // Ingestion stores the complete article in one result.\n maxLength: Number.MAX_SAFE_INTEGER,\n});\n\ntype FeedFields = {\n copyright?: string;\n language?: string;\n managingEditor?: string;\n webMaster?: string;\n lastBuildDate?: string;\n};\n\ntype ItemFields = {\n category?: string;\n creator?: string;\n enclosure?: Parser.Enclosure;\n guid?: string;\n author?: string;\n 'content:encoded'?: string;\n};\n\nconst rssParser = new Parser<FeedFields, ItemFields>({\n customFields: {\n feed: ['copyright', 'language', 'managingEditor', 'webMaster'],\n item: ['category', 'creator', 'enclosure', 'guid'],\n },\n});\n\nasync function parseRSSFeed(feedUrl: string) {\n try {\n console.log(`Fetching RSS feed: ${feedUrl}`);\n const feed = await rssParser.parseURL(feedUrl);\n\n return {\n title: feed.title || '',\n description: feed.description || '',\n link: feed.link || '',\n language: feed.language || 'en',\n lastBuildDate: feed.lastBuildDate || new Date().toISOString(),\n items: feed.items.map((item) => ({\n title: item.title || '',\n description: item.content || item.summary || item.contentSnippet || '',\n link: item.link || '',\n pubDate: item.pubDate || item.isoDate || '',\n author: item.creator || item.author || '',\n categories: Array.isArray(item.categories)\n ? item.categories\n : item.category\n ? [item.category]\n : [],\n guid: item.guid || item.guid || '',\n contentEncoded: item['content:encoded'] || item.content || '',\n })),\n };\n } catch (error) {\n console.error(`Failed to parse RSS feed ${feedUrl}:`, error);\n throw new Error(\n `RSS parsing failed: ${error instanceof Error ? error.message : 'Unknown error'}`,\n );\n }\n}\n\nasync function fetchArticleContent(url: string): Promise<string> {\n try {\n console.log(`Fetching full article content from: ${url}`);\n\n const article = await webFetch.execute(\n { url },\n { toolCallId: url, messages: [], context: {} },\n );\n assert.ok('type' in article, 'Web fetch must return a complete result');\n if (article.type === 'error') {\n throw new Error(article.message);\n }\n\n // Combine title and content for comprehensive text\n const fullContent = article.title\n ? `${article.title}\\n\\n${article.content}`\n : article.content;\n\n // Ensure we have meaningful content (at least 200 characters for full articles)\n if (fullContent.length < 200) {\n throw new Error(\n `Extracted content too short (${fullContent.length} chars) - likely failed to find main content`,\n );\n }\n\n console.log(\n `\u2713 Successfully extracted article: \"${article.title ?? ''}\" (${fullContent.length} chars, ~${Math.round(fullContent.split(' ').length)} words)`,\n );\n\n return fullContent;\n } catch (error) {\n console.warn(\n `Failed to fetch article content from ${url}:`,\n error instanceof Error ? error.message : error,\n );\n return ''; // Return empty string on failure rather than throwing\n }\n}\nexport function rss(\n feedUrl: string,\n options: {\n maxItems?: number;\n fetchFullArticles?: boolean;\n } = {},\n) {\n const sourceId = `rss:${feedUrl}`;\n const { maxItems = 50, fetchFullArticles = false } = options;\n\n return {\n sourceId,\n instructions: `You answer questions about articles and content from the RSS feed: ${feedUrl}.\n Always cite the article title and link when referencing specific content.\n The feed contains recent articles, blog posts, and news items.\n When referencing content, include the publication date and author when available.\n ${fetchFullArticles ? 'Full article content has been extracted from the original links for comprehensive analysis.' : 'Content includes RSS summaries and descriptions.'}`,\n sources: async function* () {\n const feed = await parseRSSFeed(feedUrl);\n // Add feed summary source\n const feedSummary = `RSS Feed: ${feed.title}\nDescription: ${feed.description}\nWebsite: ${feed.link}\nLanguage: ${feed.language}\nLast Updated: ${feed.lastBuildDate}\nTotal Items: ${feed.items.length}\n\nThis feed provides: ${feed.description}`;\n yield {\n id: 'feed-info',\n content: async () => feedSummary,\n };\n\n // Individual article sources (limit to maxItems)\n const itemsToProcess = feed.items.slice(0, maxItems);\n for (const item of itemsToProcess) {\n const documentId =\n item.guid || item.link || `${item.title}-${item.pubDate}`;\n yield {\n id: documentId,\n content: async () => {\n // Try full article fetch if enabled, fallback to RSS content\n let articleContent = item.contentEncoded || item.description;\n if (fetchFullArticles && item.link) {\n const fullContent = await fetchArticleContent(item.link);\n if (fullContent && fullContent.length > articleContent.length) {\n articleContent = fullContent;\n }\n }\n return `Title: ${item.title}\nAuthor: ${item.author}\nPublished: ${item.pubDate}\nCategories: ${item.categories.join(', ')}\nLink: ${item.link}\n${fetchFullArticles ? 'Full Article Content:' : 'Content:'}\n${articleContent}\n\nSummary: ${item.title} - ${item.description}`;\n },\n };\n }\n },\n };\n}\n", "import fg from 'fast-glob';\nimport { opendir, readFile, stat } from 'node:fs/promises';\nimport { join } from 'node:path';\n\nimport type { Connector } from './connector.ts';\n\nexport function repo(\n dir: string,\n extensions: string[],\n ingestWhen: Connector['ingestWhen'],\n): Connector {\n const sourceId = `repo:${dir}`;\n return {\n sourceId,\n ingestWhen,\n sources: async function* () {\n const paths = await collectFiles(dir, extensions);\n for await (const path of paths) {\n const maxSize = 3 * 1024; // 3KB\n const st = await stat(path);\n if (st.size > maxSize) {\n continue;\n }\n yield {\n id: path,\n metadata: { repo: dir },\n content: () => readFile(path, 'utf8').catch(() => ''),\n };\n }\n },\n };\n}\n\nexport async function* findAllGitRepos(root: string) {\n const stack = [root];\n const skip = [\n 'node_modules',\n 'Library',\n 'Applications',\n 'Pictures',\n 'Movies',\n 'Music',\n 'Downloads',\n '.cache',\n '.npm',\n '.pnpm',\n 'development',\n ];\n while (stack.length) {\n const folder = stack.pop()!;\n const isGitRepo = await stat(join(folder, '.git'))\n .then((st) => st.isDirectory() || st.isFile())\n .catch(() => false);\n if (isGitRepo) {\n yield folder;\n continue;\n }\n const dir = await opendir(folder);\n for await (const dirent of dir) {\n if (\n dirent.isDirectory() &&\n !dirent.isSymbolicLink() &&\n !skip.includes(dirent.name) &&\n !dirent.name.startsWith('.')\n ) {\n stack.push(join(folder, dirent.name));\n }\n }\n }\n}\n\nfunction detectRepoTooling(repo: string) {\n const tools: string[] = [];\n return tools;\n}\n\nasync function gitignore(gitignoreFile: string) {\n const content = await readFile(gitignoreFile, 'utf8').catch(() => '');\n if (!content) {\n return [];\n }\n const patterns = content\n .split('\\n')\n .map((line) => line.trim())\n .filter((line) => line && !line.startsWith('#'));\n return patterns;\n}\n\nexport async function ignorePatterns(repo: string) {\n return [\n // Inherit repo-specific ignore patterns\n ...(await gitignore(join(repo, '.gitignore'))),\n\n // Package managers & dependency dirs\n 'node_modules/**',\n '**/node_modules/**',\n '**/.pnpm/**',\n '**/.npm/**',\n '**/.yarn/**',\n '**/vendor/**', // PHP / Go modules (when vendored)\n '**/3rdparty/**',\n\n // Version control + VCS metadata\n '**/.git/**',\n '**/.svn/**',\n '**/.hg/**',\n\n // OS / system junk\n '**/.DS_Store',\n '**/Thumbs.db',\n '**/Library/**',\n '**/Applications/**',\n '**/Pictures/**',\n '**/Movies/**',\n '**/Music/**',\n '**/Downloads/**',\n '**/.cache/**',\n\n // Environment / secrets (explicit)\n '**/.env',\n '**/.env.*',\n\n // Lockfiles & generated dependency state\n '**/*.lock',\n '**/yarn.lock',\n '**/package-lock.json',\n '**/pnpm-lock.yaml',\n\n // Build / compilation outputs\n '**/dist/**',\n '**/debug/**',\n '**/build/**',\n '**/out/**',\n '**/target/**', // Rust / JVM\n '**/bin/**',\n '**/obj/**',\n '**/classes/**',\n\n // Framework / tool specific build artifacts\n '**/.next/**',\n '**/.vercel/**',\n '**/.turbo/**',\n '**/.docusaurus/**',\n '**/.vite/**',\n '**/.parcel-cache/**',\n '**/.rollup.cache/**',\n '**/.vuepress/**',\n 'cdk.out/**',\n\n // Infra & deployment tooling\n '**/.serverless/**',\n '**/.terraform/**',\n '**/.terragrunt-cache/**',\n '**/.pulumi/**',\n\n // Coverage & testing caches\n '**/coverage/**',\n '**/.nyc_output/**',\n '**/jest-cache/**',\n '**/.pytest_cache/**',\n\n // Language / tooling caches\n '**/__pycache__/**',\n '**/.mypy_cache/**',\n '**/.tox/**',\n '**/.gradle/**',\n '**/.mvn/**',\n '**/.eslintcache',\n '**/.stylelintcache',\n\n // IDE / editor configs + history (we don't want to embed these)\n '**/.idea/**',\n '**/.vscode/**',\n '**/.fleet/**',\n '**/.history/**',\n\n // Virtual environments\n '**/.venv/**',\n '**/venv/**',\n ];\n}\n\nexport async function collectFiles(\n repo: string,\n extensions: string[],\n): Promise<AsyncIterable<string>> {\n const exts = extensions.map((ext) => ext.replace(/^\\./, ''));\n const entries = fg.stream(\n extensions.length > 1 ? `**/*.{${exts.join(',')}}` : `**/*.${exts[0]}`,\n {\n dot: false,\n onlyFiles: true,\n unique: true,\n absolute: true,\n cwd: repo,\n ignore: await ignorePatterns(repo),\n },\n );\n return (async function* () {\n for await (const entry of entries) {\n if (typeof entry !== 'string') {\n throw new TypeError('fast-glob yielded a non-path entry');\n }\n yield entry;\n }\n })();\n}\n"],
|
|
5
|
-
"mappings": ";AAAA,SAAS,aAAa;AA0BtB,SAAS,UAAU,OAAkC;AACnD,SACE,OAAO,UAAU,YACjB,UAAU,QACV,cAAc,SACd,OAAO,MAAM,aAAa,YAC1B,UAAU,UACT,MAAM,SAAS,QAAQ,OAAO,MAAM,SAAS,aAC9C,UAAU,UACT,MAAM,SAAS,QAAQ,OAAO,MAAM,SAAS,aAC9C,gBAAgB,SAChB,OAAO,MAAM,eAAe,YAC5B,kBAAkB,SAClB,OAAO,MAAM,iBAAiB,YAC9B,cAAc,SACd,OAAO,MAAM,aAAa,YAC1B,WAAW,SACX,OAAO,MAAM,UAAU,aACvB,gBAAgB,SAChB,OAAO,MAAM,eAAe;AAEhC;AAEA,SAAS,GAAG,MAAc;AACxB,QAAM,CAAC,OAAOA,OAAM,GAAG,QAAQ,IAAI,KAAK,MAAM,GAAG;AACjD,SAAO;AAAA,IACL,UAAU,YAAY;AACpB,YAAM,MAAM,gCAAgC,KAAK,IAAIA,KAAI,aAAa,SAAS,KAAK,GAAG,CAAC;AACxF,YAAM,MAAM,MAAM,MAAM,GAAG;AAC3B,YAAM,OAAgB,MAAM,IAAI,KAAK;AACrC,UACE,OAAO,SAAS,YAChB,SAAS,QACT,EAAE,aAAa,SACf,OAAO,KAAK,YAAY,UACxB;AACA,cAAM,IAAI,MAAM,uCAAuC,IAAI,EAAE;AAAA,MAC/D;AACA,aAAO,KAAK,KAAK,OAAO;AAAA,IAC1B;AAAA,IACA,SAAS,OAAOA,OAAc,OAA4B,CAAC,MAAM;AAC/D,YAAM,CAACC,QAAO,QAAQ,IAAID,MAAK,MAAM,GAAG;AACxC,YAAM,UAAU;AAChB,YAAM,WAAW;AACjB,YAAM,WAAsB,CAAC;AAC7B,UAAI,OAAO;AAEX,eAAS,OAAO,GAAG,QAAQ,YAAY,CAAC,MAAM,QAAQ;AACpD,cAAM,MAAM,gCAAgCC,MAAK,IAAI,QAAQ,sBAAsB,OAAO,SAAS,IAAI;AACvG,cAAM,MAAM,MAAM,MAAM,KAAK;AAAA,UAC3B,SAAS,EAAE,QAAQ,8BAA8B;AAAA,QACnD,CAAC;AACD,YAAI,CAAC,IAAI,GAAI;AACb,cAAM,WAAoB,MAAM,IAAI,KAAK;AACzC,YAAI,CAAC,MAAM,QAAQ,QAAQ,KAAK,CAAC,SAAS,MAAM,SAAS,GAAG;AAC1D,gBAAM,IAAI,MAAM,2CAA2CD,KAAI,EAAE;AAAA,QACnE;AACA,YAAI,CAAC,SAAS,OAAQ;AACtB,mBAAW,OAAO,UAAU;AAC1B,cAAI,CAAC,KAAK,iBAAiB,IAAI,MAAO;AACtC,cAAI,CAAC,KAAK,qBAAqB,IAAI,WAAY;AAE/C,mBAAS,KAAK,GAAG;AACjB,cAAI,KAAK,YAAY,IAAI,aAAa,KAAK,UAAU;AACnD,gBAAI,KAAK,cAAc,OAAO;AAC5B,uBAAS,IAAI;AAAA,YACf;AACA,mBAAO;AACP;AAAA,UACF;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAEO,IAAM,SAAS;AAAA,EACpB,KAAK,UAA6B;AAChC,UAAM,WAAW,eAAe,QAAQ;AACxC,UAAM,aAAa;AACnB,WAAO;AAAA,MACL;AAAA,MACA,SAAS,mBAAmB;AAC1B,cAAM,EAAE,UAAAE,UAAS,IAAI,GAAG,QAAQ;AAChC,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,MAAMA,UAAS;AAAA,QAC1B;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,SAAS,CAACF,OAAc,UAA+B,CAAC,MAAiB;AACvE,UAAM,WAAW,mBAAmBA,KAAI;AACxC,WAAO;AAAA,MACL;AAAA,MACA,SAAS,mBAAmB;AAC1B,cAAM,WAAW,MAAM,GAAGA,KAAI,EAAE,QAAQA,OAAM,OAAO;AACrD,mBAAW,OAAO,UAAU;AAC1B,gBAAM;AAAA,YACJ,IAAI,GAAGA,KAAI,YAAY,IAAI,QAAQ;AAAA,YACnC,SAAS,YAAY;AACnB,oBAAM,OAAO,IAAI,QAAQ,IAAI;AAC7B,oBAAM,OAAO,IAAI,QAAQ;AACzB,qBAAO,YAAY,IAAI;AAAA,OAAU,IAAI,QAAQ;AAAA,gBAAmB,IAAI,YAAY;AAAA,cAAiB,IAAI,UAAU;AAAA,OAAU,IAAI,QAAQ;AAAA,SAAY,IAAI,KAAK;AAAA,cAAiB,IAAI,UAAU;AAAA;AAAA,EAAO,IAAI;AAAA,YACtM;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,CACJ,SACA,SASc;AACd,UAAM,WAAW,eAAe,OAAO;AACvC,UAAM;AAAA,MACJ,WAAW;AAAA,QACT;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA;AAAA,MACA,oBAAoB;AAAA,MACpB,oBAAoB;AAAA,MACpB;AAAA,MACA;AAAA,MACA;AAAA,IACF,IAAI;AAEJ,mBAAe,kBAAmC;AAChD,YAAM,OAAO;AAAA,QACX;AAAA,QACA,GAAI,SAAS,CAAC,MAAM,MAAM,IAAI,CAAC;AAAA,QAC/B,GAAG,SAAS,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAAA,QACpC,GAAG,SAAS,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAAA,QACpC,GAAI,oBAAoB,CAAC,sBAAsB,IAAI,CAAC;AAAA,QACpD,GAAI,oBAAoB,CAAC,sBAAsB,IAAI,CAAC;AAAA,QACpD;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,YAAM,EAAE,OAAO,IAAI,MAAM,MAAM,OAAO,MAAM;AAAA,QAC1C,KAAK,cACD,EAAE,GAAG,QAAQ,KAAK,cAAc,YAAY,IAC5C,QAAQ;AAAA,QACZ,QAAQ;AAAA,QACR,QAAQ;AAAA,MACV,CAAC;AACD,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,SAAS,mBAAmB;AAC1B,cAAM,UAAU,MAAM,gBAAgB;AACtC,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AAAA,QACvB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC3NA,SAAS,oBAAoB;AAEtB,SAAS,OAAO,QAAgB;AACrC,QAAM,eAAe,IAAI,aAAa,EAAE,OAAO,CAAC;AAChD,QAAM,WAAW;AACjB,SAAO;AAAA,IACL;AAAA,IACA,cACE;AAAA,IACF,SAAS,YAAY;AACnB,YAAM,OAAO,MAAM,aAAa;AAChC,YAAM,SAAS,MAAM,KAAK,eAAe;AACzC,aAAO,QAAQ;AAAA,QACb,OAAO,MAAM,IAAI,OAAO,OAAO;AAC7B,gBAAM,QAAQ,MAAM,GAAG;AACvB,iBAAO;AAAA,YACL,IAAI,GAAG;AAAA,YACP,SAAS,YACP,UAAU,GAAG,KAAK;AAAA,eAAkB,GAAG,eAAe,gBAAgB;AAAA,UAAa,OAAO,QAAQ,SAAS;AAAA,MAAS,GAAG,EAAE;AAAA,UAC7H;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;;;ACxBA,OAAO,QAAQ;AACf,OAAO,YAAY;AACnB,SAAS,UAAU,YAAY;AAC/B,SAAS,SAAS,MAAM,UAAU,eAAe;AAOjD,IAAM,iBAAiB,oBAAI,IAAuC;AAKlE,eAAe,yBACb,YACoC;AACpC,QAAM,eAAe,QAAQ,UAAU;AAEvC,MAAI,eAAe,IAAI,YAAY,GAAG;AACpC,WAAO,eAAe,IAAI,YAAY;AAAA,EACxC;AAEA,QAAM,KAAK,OAAO;AAGlB,QAAM,eAAe,aAAa,MAAM,GAAG;AAC3C,QAAM,iBAA2B,CAAC;AAGlC,WAAS,IAAI,GAAG,KAAK,aAAa,QAAQ,KAAK;AAC7C,UAAM,cAAc,aAAa,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG;AACrD,UAAM,gBAAgB,KAAK,aAAa,YAAY;AAEpD,QAAI;AACF,YAAM,KAAK,aAAa;AACxB,qBAAe,KAAK,aAAa;AAAA,IACnC,QAAQ;AAAA,IAER;AAAA,EACF;AAGA,aAAW,iBAAiB,gBAAgB;AAC1C,QAAI;AACF,YAAM,UAAU,MAAM,SAAS,eAAe,MAAM;AACpD,YAAM,WAAW,QACd,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,QAAQ,CAAC,KAAK,WAAW,GAAG,CAAC;AAEjD,SAAG,IAAI,QAAQ;AAAA,IACjB,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,iBAAe,IAAI,cAAc,EAAE;AACnC,SAAO;AACT;AAKA,eAAe,iBACb,KACA,SACmB;AAEnB,QAAM,WAAW,MAAM,GAAG,SAAS;AAAA,IACjC,KAAK;AAAA,IACL,UAAU;AAAA,IACV,WAAW;AAAA,IACX;AAAA,IACA,qBAAqB;AAAA,IACrB,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF,CAAC;AAED,MAAI,SAAS,WAAW,GAAG;AACzB,WAAO,CAAC;AAAA,EACV;AAGA,QAAM,aAAa,oBAAI,IAAsB;AAE7C,aAAW,QAAQ,UAAU;AAC3B,UAAM,MAAM,QAAQ,IAAI;AACxB,QAAI,CAAC,WAAW,IAAI,GAAG,GAAG;AACxB,iBAAW,IAAI,KAAK,CAAC,CAAC;AAAA,IACxB;AACA,eAAW,IAAI,GAAG,EAAG,KAAK,IAAI;AAAA,EAChC;AAEA,QAAM,gBAA0B,CAAC;AAGjC,aAAW,CAAC,KAAK,KAAK,KAAK,YAAY;AACrC,UAAM,eAAe,MAAM,yBAAyB,GAAG;AAEvD,eAAW,QAAQ,OAAO;AACxB,UAAI,CAAC,aAAa,QAAQ,SAAS,KAAK,IAAI,CAAC,GAAG;AAC9C,sBAAc,KAAK,IAAI;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAEA,SAAO,cAAc,KAAK;AAC5B;AAEO,SAAS,MACd,SACA,SAKW;AACX,QAAM,WAAW,QAAQ,OAAO;AAChC,SAAO;AAAA,IACL;AAAA,IACA,YAAY,SAAS;AAAA,IACrB,cAAc,SAAS;AAAA,IACvB,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM;AAAA,QAClB,SAAS,OAAO,QAAQ,IAAI;AAAA,QAC5B;AAAA,MACF;AACA,iBAAW,YAAY,OAAO;AAC5B,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,MAAM,SAAS,UAAU,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,QAC1D;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AClJA,SAAS,MAAM,YAAAG,iBAAgB;AAC/B,SAAS,aAAa,wBAAwB;AAIvC,SAAS,IAAI,SAA4B;AAC9C,QAAM,WAAW,OAAO,OAAO;AAE/B,SAAO;AAAA,IACL;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM,MAAM;AAAA,QACxB,KAAK,SAAS,EAAE,SAAS,CAAC,sBAAsB,YAAY,EAAE,CAAC;AAAA,MACjE;AACA,iBAAW,QAAQ,OAAO;AACxB,YAAI,CAAC,KAAK,YAAY,EAAE,SAAS,MAAM,EAAG;AAC1C,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AACnB,kBAAM,SAAS,MAAMA,UAAS,IAAI;AAClC,kBAAMC,OAAM,MAAM,iBAAiB,IAAI,WAAW,MAAM,CAAC;AACzD,kBAAM,EAAE,KAAK,IAAI,MAAM,YAAYA,MAAK,EAAE,YAAY,KAAK,CAAC;AAC5D,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,QAAQ,QAA2B;AACjD,QAAM,QAAQ,eAAe,KAAK,MAAM;AACxC,QAAM,WAAW,OAAO,QAAQ,QAAQ,MAAM,IAAI,MAAM;AAExD,SAAO;AAAA,IACL;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM;AAAA,QACJ,IAAI;AAAA,QACJ,SAAS,YAAY;AACnB,cAAI;AAEJ,cAAI,OAAO;AACT,kBAAM,WAAW,MAAM,MAAM,MAAM;AACnC,gBAAI,CAAC,SAAS,IAAI;AAChB,oBAAM,IAAI;AAAA,gBACR,QAAQ,SAAS,MAAM,KAAK,SAAS,UAAU;AAAA,cACjD;AAAA,YACF;AACA,qBAAS,IAAI,WAAW,MAAM,SAAS,YAAY,CAAC;AAAA,UACtD,OAAO;AACL,kBAAM,aAAa,MAAMD,UAAS,MAAM;AACxC,qBAAS,IAAI,WAAW,WAAW,MAAM;AAAA,UAC3C;AAEA,gBAAMC,OAAM,MAAM,iBAAiB,IAAI,WAAW,MAAM,CAAC;AACzD,gBAAM,EAAE,KAAK,IAAI,MAAM,YAAYA,MAAK,EAAE,YAAY,KAAK,CAAC;AAC5D,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC9DA,SAAS,uCAAuC;AAChD,OAAO,YAAY;AACnB,OAAO,YAAY;AAEnB,IAAM,WAAW,gCAAgC;AAAA;AAAA,EAE/C,WAAW,OAAO;AACpB,CAAC;AAmBD,IAAM,YAAY,IAAI,OAA+B;AAAA,EACnD,cAAc;AAAA,IACZ,MAAM,CAAC,aAAa,YAAY,kBAAkB,WAAW;AAAA,IAC7D,MAAM,CAAC,YAAY,WAAW,aAAa,MAAM;AAAA,EACnD;AACF,CAAC;AAED,eAAe,aAAa,SAAiB;AAC3C,MAAI;AACF,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,UAAM,OAAO,MAAM,UAAU,SAAS,OAAO;AAE7C,WAAO;AAAA,MACL,OAAO,KAAK,SAAS;AAAA,MACrB,aAAa,KAAK,eAAe;AAAA,MACjC,MAAM,KAAK,QAAQ;AAAA,MACnB,UAAU,KAAK,YAAY;AAAA,MAC3B,eAAe,KAAK,kBAAiB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC5D,OAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,QAC/B,OAAO,KAAK,SAAS;AAAA,QACrB,aAAa,KAAK,WAAW,KAAK,WAAW,KAAK,kBAAkB;AAAA,QACpE,MAAM,KAAK,QAAQ;AAAA,QACnB,SAAS,KAAK,WAAW,KAAK,WAAW;AAAA,QACzC,QAAQ,KAAK,WAAW,KAAK,UAAU;AAAA,QACvC,YAAY,MAAM,QAAQ,KAAK,UAAU,IACrC,KAAK,aACL,KAAK,WACH,CAAC,KAAK,QAAQ,IACd,CAAC;AAAA,QACP,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,QAChC,gBAAgB,KAAK,iBAAiB,KAAK,KAAK,WAAW;AAAA,MAC7D,EAAE;AAAA,IACJ;AAAA,EACF,SAAS,OAAO;AACd,YAAQ,MAAM,4BAA4B,OAAO,KAAK,KAAK;AAC3D,UAAM,IAAI;AAAA,MACR,uBAAuB,iBAAiB,QAAQ,MAAM,UAAU,eAAe;AAAA,IACjF;AAAA,EACF;AACF;AAEA,eAAe,oBAAoB,KAA8B;AAC/D,MAAI;AACF,YAAQ,IAAI,uCAAuC,GAAG,EAAE;AAExD,UAAM,UAAU,MAAM,SAAS;AAAA,MAC7B,EAAE,IAAI;AAAA,MACN,EAAE,YAAY,KAAK,UAAU,CAAC,GAAG,SAAS,CAAC,EAAE;AAAA,IAC/C;AACA,WAAO,GAAG,UAAU,SAAS,yCAAyC;AACtE,QAAI,QAAQ,SAAS,SAAS;AAC5B,YAAM,IAAI,MAAM,QAAQ,OAAO;AAAA,IACjC;AAGA,UAAM,cAAc,QAAQ,QACxB,GAAG,QAAQ,KAAK;AAAA;AAAA,EAAO,QAAQ,OAAO,KACtC,QAAQ;AAGZ,QAAI,YAAY,SAAS,KAAK;AAC5B,YAAM,IAAI;AAAA,QACR,gCAAgC,YAAY,MAAM;AAAA,MACpD;AAAA,IACF;AAEA,YAAQ;AAAA,MACN,2CAAsC,QAAQ,SAAS,EAAE,MAAM,YAAY,MAAM,YAAY,KAAK,MAAM,YAAY,MAAM,GAAG,EAAE,MAAM,CAAC;AAAA,IACxI;AAEA,WAAO;AAAA,EACT,SAAS,OAAO;AACd,YAAQ;AAAA,MACN,wCAAwC,GAAG;AAAA,MAC3C,iBAAiB,QAAQ,MAAM,UAAU;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AACF;AACO,SAAS,IACd,SACA,UAGI,CAAC,GACL;AACA,QAAM,WAAW,OAAO,OAAO;AAC/B,QAAM,EAAE,WAAW,IAAI,oBAAoB,MAAM,IAAI;AAErD,SAAO;AAAA,IACL;AAAA,IACA,cAAc,sEAAsE,OAAO;AAAA;AAAA;AAAA;AAAA,QAIvF,oBAAoB,gGAAgG,kDAAkD;AAAA,IAC1K,SAAS,mBAAmB;AAC1B,YAAM,OAAO,MAAM,aAAa,OAAO;AAEvC,YAAM,cAAc,aAAa,KAAK,KAAK;AAAA,eAClC,KAAK,WAAW;AAAA,WACpB,KAAK,IAAI;AAAA,YACR,KAAK,QAAQ;AAAA,gBACT,KAAK,aAAa;AAAA,eACnB,KAAK,MAAM,MAAM;AAAA;AAAA,sBAEV,KAAK,WAAW;AAChC,YAAM;AAAA,QACJ,IAAI;AAAA,QACJ,SAAS,YAAY;AAAA,MACvB;AAGA,YAAM,iBAAiB,KAAK,MAAM,MAAM,GAAG,QAAQ;AACnD,iBAAW,QAAQ,gBAAgB;AACjC,cAAM,aACJ,KAAK,QAAQ,KAAK,QAAQ,GAAG,KAAK,KAAK,IAAI,KAAK,OAAO;AACzD,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AAEnB,gBAAI,iBAAiB,KAAK,kBAAkB,KAAK;AACjD,gBAAI,qBAAqB,KAAK,MAAM;AAClC,oBAAM,cAAc,MAAM,oBAAoB,KAAK,IAAI;AACvD,kBAAI,eAAe,YAAY,SAAS,eAAe,QAAQ;AAC7D,iCAAiB;AAAA,cACnB;AAAA,YACF;AACA,mBAAO,UAAU,KAAK,KAAK;AAAA,UAC7B,KAAK,MAAM;AAAA,aACR,KAAK,OAAO;AAAA,cACX,KAAK,WAAW,KAAK,IAAI,CAAC;AAAA,QAChC,KAAK,IAAI;AAAA,EACf,oBAAoB,0BAA0B,UAAU;AAAA,EACxD,cAAc;AAAA;AAAA,WAEL,KAAK,KAAK,MAAM,KAAK,WAAW;AAAA,UACjC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;ACxKA,OAAOC,SAAQ;AACf,SAAS,SAAS,YAAAC,WAAU,QAAAC,aAAY;AACxC,SAAS,QAAAC,aAAY;AAId,SAAS,KACd,KACA,YACA,YACW;AACX,QAAM,WAAW,QAAQ,GAAG;AAC5B,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM,aAAa,KAAK,UAAU;AAChD,uBAAiB,QAAQ,OAAO;AAC9B,cAAM,UAAU,IAAI;AACpB,cAAM,KAAK,MAAMD,MAAK,IAAI;AAC1B,YAAI,GAAG,OAAO,SAAS;AACrB;AAAA,QACF;AACA,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,UAAU,EAAE,MAAM,IAAI;AAAA,UACtB,SAAS,MAAMD,UAAS,MAAM,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,QACtD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,gBAAuB,gBAAgB,MAAc;AACnD,QAAM,QAAQ,CAAC,IAAI;AACnB,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO,MAAM,QAAQ;AACnB,UAAM,SAAS,MAAM,IAAI;AACzB,UAAM,YAAY,MAAMC,MAAKC,MAAK,QAAQ,MAAM,CAAC,EAC9C,KAAK,CAAC,OAAO,GAAG,YAAY,KAAK,GAAG,OAAO,CAAC,EAC5C,MAAM,MAAM,KAAK;AACpB,QAAI,WAAW;AACb,YAAM;AACN;AAAA,IACF;AACA,UAAM,MAAM,MAAM,QAAQ,MAAM;AAChC,qBAAiB,UAAU,KAAK;AAC9B,UACE,OAAO,YAAY,KACnB,CAAC,OAAO,eAAe,KACvB,CAAC,KAAK,SAAS,OAAO,IAAI,KAC1B,CAAC,OAAO,KAAK,WAAW,GAAG,GAC3B;AACA,cAAM,KAAKA,MAAK,QAAQ,OAAO,IAAI,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AACF;AAOA,eAAe,UAAU,eAAuB;AAC9C,QAAM,UAAU,MAAMC,UAAS,eAAe,MAAM,EAAE,MAAM,MAAM,EAAE;AACpE,MAAI,CAAC,SAAS;AACZ,WAAO,CAAC;AAAA,EACV;AACA,QAAM,WAAW,QACd,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,QAAQ,CAAC,KAAK,WAAW,GAAG,CAAC;AACjD,SAAO;AACT;AAEA,eAAsB,eAAeC,OAAc;AACjD,SAAO;AAAA;AAAA,IAEL,GAAI,MAAM,UAAUC,MAAKD,OAAM,YAAY,CAAC;AAAA;AAAA,IAG5C;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,EACF;AACF;AAEA,eAAsB,aACpBA,OACA,YACgC;AAChC,QAAM,OAAO,WAAW,IAAI,CAAC,QAAQ,IAAI,QAAQ,OAAO,EAAE,CAAC;AAC3D,QAAM,UAAUE,IAAG;AAAA,IACjB,WAAW,SAAS,IAAI,SAAS,KAAK,KAAK,GAAG,CAAC,MAAM,QAAQ,KAAK,CAAC,CAAC;AAAA,IACpE;AAAA,MACE,KAAK;AAAA,MACL,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,KAAKF;AAAA,MACL,QAAQ,MAAM,eAAeA,KAAI;AAAA,IACnC;AAAA,EACF;AACA,UAAQ,mBAAmB;AACzB,qBAAiB,SAAS,SAAS;AACjC,UAAI,OAAO,UAAU,UAAU;AAC7B,cAAM,IAAI,UAAU,oCAAoC;AAAA,MAC1D;AACA,YAAM;AAAA,IACR;AAAA,EACF,GAAG;AACL;",
|
|
4
|
+
"sourcesContent": ["import { execa } from 'execa';\n\nimport type { Connector } from './connector.ts';\n\ninterface ReleaseFetchOptions {\n /** Stop fetching when this tag is encountered (latest-first order). */\n untilTag?: string;\n /** Include the matching untilTag release (default true). */\n inclusive?: boolean;\n /** Include draft releases (default false). */\n includeDrafts?: boolean;\n /** Include prereleases (default false). */\n includePrerelease?: boolean;\n}\n\ninterface Release {\n tag_name: string;\n name: string | null;\n body: string | null;\n updated_at: string;\n published_at: string;\n html_url: string;\n draft: boolean;\n prerelease: boolean;\n}\n\nfunction isRelease(value: unknown): value is Release {\n return (\n typeof value === 'object' &&\n value !== null &&\n 'tag_name' in value &&\n typeof value.tag_name === 'string' &&\n 'name' in value &&\n (value.name === null || typeof value.name === 'string') &&\n 'body' in value &&\n (value.body === null || typeof value.body === 'string') &&\n 'updated_at' in value &&\n typeof value.updated_at === 'string' &&\n 'published_at' in value &&\n typeof value.published_at === 'string' &&\n 'html_url' in value &&\n typeof value.html_url === 'string' &&\n 'draft' in value &&\n typeof value.draft === 'boolean' &&\n 'prerelease' in value &&\n typeof value.prerelease === 'boolean'\n );\n}\n\nfunction fs(path: string) {\n const [owner, repo, ...filePath] = path.split('/');\n return {\n readFile: async () => {\n const url = `https://api.github.com/repos/${owner}/${repo}/contents/${filePath.join('/')}`;\n const res = await fetch(url);\n const data: unknown = await res.json();\n if (\n typeof data !== 'object' ||\n data === null ||\n !('content' in data) ||\n typeof data.content !== 'string'\n ) {\n throw new Error(`GitHub returned no file content for ${path}`);\n }\n return atob(data.content);\n },\n release: async (repo: string, opts: ReleaseFetchOptions = {}) => {\n const [owner, repoName] = repo.split('/');\n const perPage = 100; // always fetch max per page for efficiency\n const maxPages = 10; // internal safety cap, not user configurable\n const releases: Release[] = [];\n let stop = false;\n\n for (let page = 1; page <= maxPages && !stop; page++) {\n const url = `https://api.github.com/repos/${owner}/${repoName}/releases?per_page=${perPage}&page=${page}`;\n const res = await fetch(url, {\n headers: { Accept: 'application/vnd.github+json' },\n });\n if (!res.ok) break;\n const pageData: unknown = await res.json();\n if (!Array.isArray(pageData) || !pageData.every(isRelease)) {\n throw new Error(`Unexpected GitHub releases response for ${repo}`);\n }\n if (!pageData.length) break;\n for (const rel of pageData) {\n if (!opts.includeDrafts && rel.draft) continue;\n if (!opts.includePrerelease && rel.prerelease) continue;\n\n releases.push(rel);\n if (opts.untilTag && rel.tag_name === opts.untilTag) {\n if (opts.inclusive === false) {\n releases.pop();\n }\n stop = true;\n break;\n }\n }\n }\n return releases;\n },\n };\n}\n\nexport const github = {\n file(filePath: string): Connector {\n const sourceId = `github:file:${filePath}`;\n const documentId = filePath;\n return {\n sourceId,\n sources: async function* () {\n const { readFile } = fs(filePath);\n yield {\n id: documentId,\n content: () => readFile(),\n };\n },\n };\n },\n /**\n * Create a connector over GitHub Releases with optional pagination & early-stop.\n * @param repo owner/repo string\n * @param options Release filtering & pagination controls\n */\n release: (repo: string, options: ReleaseFetchOptions = {}): Connector => {\n const sourceId = `github:releases:${repo}`;\n return {\n sourceId,\n sources: async function* () {\n const releases = await fs(repo).release(repo, options);\n for (const rel of releases) {\n yield {\n id: `${repo}:release:${rel.tag_name}`,\n content: async () => {\n const name = rel.name || rel.tag_name;\n const body = rel.body || '';\n return `Release: ${name}\\nTag: ${rel.tag_name}\\nPublished at: ${rel.published_at}\\nUpdated at: ${rel.updated_at}\\nURL: ${rel.html_url}\\nDraft: ${rel.draft}\\nPrerelease: ${rel.prerelease}\\n\\n${body}`;\n },\n };\n }\n },\n };\n },\n /**\n * Create a connector over a GitHub repository file tree using gitingest to produce a markdown digest.\n * @param repoUrl full https://github.com/OWNER/REPO or .../tree/BRANCH/subdir URL\n */\n repo: (\n repoUrl: string,\n opts: {\n includes: string[];\n excludes?: string[];\n branch?: string;\n includeGitignored?: boolean;\n includeSubmodules?: boolean;\n githubToken?: string;\n ingestWhen?: 'never' | 'contentChanged';\n },\n ): Connector => {\n const sourceId = `github:repo:${repoUrl}`;\n const {\n excludes = [\n '**/node_modules/**',\n '**/dist/**',\n '**/coverage/**',\n '**/*.test.ts',\n '**/*.test.tsx',\n '**/dist/**',\n '**/.git/**',\n '**/.github/**',\n '**/.vscode/**',\n '**/.idea/**',\n '**/build/**',\n '**/out/**',\n '**/vendor/**',\n '**/__tests__/**',\n '**/*.d.ts',\n ],\n branch,\n includeGitignored = false,\n includeSubmodules = false,\n githubToken,\n ingestWhen,\n includes,\n } = opts;\n\n async function gitingestDigest(): Promise<string> {\n const args = [\n 'gitingest',\n ...(branch ? ['-b', branch] : []),\n ...includes.flatMap((p) => ['-i', p]),\n ...excludes.flatMap((p) => ['-e', p]),\n ...(includeGitignored ? ['--include-gitignored'] : []),\n ...(includeSubmodules ? ['--include-submodules'] : []),\n repoUrl,\n '-o',\n '-',\n ];\n const { stdout } = await execa('uvx', args, {\n env: githubToken\n ? { ...process.env, GITHUB_TOKEN: githubToken }\n : process.env,\n stdout: 'pipe',\n stderr: 'inherit',\n });\n return stdout;\n }\n\n return {\n sourceId,\n ingestWhen,\n sources: async function* () {\n const content = await gitingestDigest();\n yield {\n id: 'digest.md',\n content: async () => content,\n };\n },\n };\n },\n};\n", "import { LinearClient } from '@linear/sdk';\n\nexport function linear(apiKey: string) {\n const linearClient = new LinearClient({ apiKey });\n const sourceId = 'linear:workspace';\n return {\n sourceId,\n instructions:\n 'You answer questions about Linear issues assigned to the user.',\n sources: async () => {\n const user = await linearClient.viewer;\n const issues = await user.assignedIssues();\n return Promise.all(\n issues.nodes.map(async (it) => {\n const state = await it.state;\n return {\n id: it.id,\n content: async () =>\n `Issue: ${it.title}\\nDescription: ${it.description || 'No description'}\\nStatus: ${state?.name || 'Unknown'}\\nID: ${it.id}`,\n };\n }),\n );\n },\n };\n}\n", "import fg from 'fast-glob';\nimport ignore from 'ignore';\nimport { readFile, stat } from 'node:fs/promises';\nimport { dirname, join, relative, resolve } from 'node:path';\n\nimport type { Connector, IngestPolicy } from './connector.ts';\n\n/**\n * Cache for gitignore patterns to avoid repeated file reads\n */\nconst gitignoreCache = new Map<string, ReturnType<typeof ignore>>();\n\n/**\n * Collect all .gitignore patterns from root directory up to target directory\n */\nasync function collectGitignorePatterns(\n targetPath: string,\n): Promise<ReturnType<typeof ignore>> {\n const absolutePath = resolve(targetPath);\n\n if (gitignoreCache.has(absolutePath)) {\n return gitignoreCache.get(absolutePath)!;\n }\n\n const ig = ignore();\n\n // Find all .gitignore files from root to target directory\n const pathSegments = absolutePath.split('/');\n const gitignoreFiles: string[] = [];\n\n // Check each level from root to target\n for (let i = 1; i <= pathSegments.length; i++) {\n const currentPath = pathSegments.slice(0, i).join('/');\n const gitignorePath = join(currentPath, '.gitignore');\n\n try {\n await stat(gitignorePath);\n gitignoreFiles.push(gitignorePath);\n } catch {\n // .gitignore doesn't exist at this level, continue\n }\n }\n\n // Read and add patterns from all .gitignore files\n for (const gitignoreFile of gitignoreFiles) {\n try {\n const content = await readFile(gitignoreFile, 'utf8');\n const patterns = content\n .split('\\n')\n .map((line) => line.trim())\n .filter((line) => line && !line.startsWith('#'));\n\n ig.add(patterns);\n } catch {\n // Skip if can't read gitignore file\n }\n }\n\n gitignoreCache.set(absolutePath, ig);\n return ig;\n}\n\n/**\n * Get all files matching the pattern while respecting .gitignore files\n */\nasync function getFilteredFiles(\n cwd: string,\n pattern: string,\n): Promise<string[]> {\n // Get all files matching the pattern without gitignore filtering first\n const allFiles = await fg(pattern, {\n dot: false,\n absolute: true,\n onlyFiles: true,\n cwd: cwd,\n followSymbolicLinks: false,\n ignore: [\n '**/node_modules/**',\n '**/.git/**',\n '**/.DS_Store',\n '**/Thumbs.db',\n '**/*.tmp',\n '**/*.temp',\n '**/coverage/**',\n '**/dist/**',\n '**/build/**',\n ],\n });\n\n if (allFiles.length === 0) {\n return [];\n }\n\n // Group files by their directory to optimize gitignore pattern collection\n const filesByDir = new Map<string, string[]>();\n\n for (const file of allFiles) {\n const dir = dirname(file);\n if (!filesByDir.has(dir)) {\n filesByDir.set(dir, []);\n }\n filesByDir.get(dir)!.push(file);\n }\n\n const filteredFiles: string[] = [];\n\n // Process each directory and filter files\n for (const [dir, files] of filesByDir) {\n const ignoreFilter = await collectGitignorePatterns(dir);\n\n for (const file of files) {\n if (!ignoreFilter.ignores(relative(cwd, file))) {\n filteredFiles.push(file);\n }\n }\n }\n\n return filteredFiles.sort();\n}\n\nexport function local(\n pattern: string,\n options: IngestPolicy & { cwd?: string } = {},\n): Connector {\n const { cwd, ...policy } = options;\n const sourceId = `glob:${pattern}`;\n return {\n ...policy,\n sourceId,\n sources: async function* () {\n const paths = await getFilteredFiles(cwd ?? process.cwd(), pattern);\n for (const filePath of paths) {\n yield {\n id: filePath,\n content: () => readFile(filePath, 'utf8').catch(() => ''),\n };\n }\n },\n };\n}\n", "import { glob, readFile } from 'node:fs/promises';\nimport { extractText, getDocumentProxy } from 'unpdf';\n\nimport type { Connector } from './connector.ts';\n\nexport function pdf(pattern: string): Connector {\n const sourceId = `pdf:${pattern}`;\n\n return {\n sourceId,\n sources: async function* () {\n const paths = await Array.fromAsync(\n glob(pattern, { exclude: ['**/node_modules/**', '**/.git/**'] }),\n );\n for (const path of paths) {\n if (!path.toLowerCase().endsWith('.pdf')) continue;\n yield {\n id: path,\n content: async () => {\n const buffer = await readFile(path);\n const pdf = await getDocumentProxy(new Uint8Array(buffer));\n const { text } = await extractText(pdf, { mergePages: true });\n return text;\n },\n };\n }\n },\n };\n}\n\nexport function pdfFile(source: string): Connector {\n const isUrl = /^https?:\\/\\//.test(source);\n const sourceId = `pdf:${isUrl ? 'url' : 'file'}:${source}`;\n\n return {\n sourceId,\n sources: async function* () {\n yield {\n id: source,\n content: async () => {\n let buffer: Uint8Array;\n\n if (isUrl) {\n const response = await fetch(source);\n if (!response.ok) {\n throw new Error(\n `HTTP ${response.status}: ${response.statusText}`,\n );\n }\n buffer = new Uint8Array(await response.arrayBuffer());\n } else {\n const fileBuffer = await readFile(source);\n buffer = new Uint8Array(fileBuffer.buffer);\n }\n\n const pdf = await getDocumentProxy(new Uint8Array(buffer));\n const { text } = await extractText(pdf, { mergePages: true });\n return text;\n },\n };\n },\n };\n}\n", "import { experimental_createWebFetchTool } from '@ai-sdk/tool-web-fetch';\nimport assert from 'node:assert/strict';\nimport Parser from 'rss-parser';\n\nconst webFetch = experimental_createWebFetchTool({\n // Ingestion stores the complete article in one result.\n maxLength: Number.MAX_SAFE_INTEGER,\n});\n\ntype FeedFields = {\n copyright?: string;\n language?: string;\n managingEditor?: string;\n webMaster?: string;\n lastBuildDate?: string;\n};\n\ntype ItemFields = {\n category?: string;\n creator?: string;\n enclosure?: Parser.Enclosure;\n guid?: string;\n author?: string;\n 'content:encoded'?: string;\n};\n\nconst rssParser = new Parser<FeedFields, ItemFields>({\n customFields: {\n feed: ['copyright', 'language', 'managingEditor', 'webMaster'],\n item: ['category', 'creator', 'enclosure', 'guid'],\n },\n});\n\nasync function parseRSSFeed(feedUrl: string) {\n try {\n console.log(`Fetching RSS feed: ${feedUrl}`);\n const feed = await rssParser.parseURL(feedUrl);\n\n return {\n title: feed.title || '',\n description: feed.description || '',\n link: feed.link || '',\n language: feed.language || 'en',\n lastBuildDate: feed.lastBuildDate || new Date().toISOString(),\n items: feed.items.map((item) => ({\n title: item.title || '',\n description: item.content || item.summary || item.contentSnippet || '',\n link: item.link || '',\n pubDate: item.pubDate || item.isoDate || '',\n author: item.creator || item.author || '',\n categories: Array.isArray(item.categories)\n ? item.categories\n : item.category\n ? [item.category]\n : [],\n guid: item.guid || item.guid || '',\n contentEncoded: item['content:encoded'] || item.content || '',\n })),\n };\n } catch (error) {\n console.error(`Failed to parse RSS feed ${feedUrl}:`, error);\n throw new Error(\n `RSS parsing failed: ${Error.isError(error) ? error.message : 'Unknown error'}`,\n { cause: error },\n );\n }\n}\n\nasync function fetchArticleContent(url: string): Promise<string> {\n try {\n console.log(`Fetching full article content from: ${url}`);\n\n const article = await webFetch.execute(\n { url },\n { toolCallId: url, messages: [], context: {} },\n );\n assert.ok('type' in article, 'Web fetch must return a complete result');\n if (article.type === 'error') {\n throw new Error(article.message);\n }\n\n // Combine title and content for comprehensive text\n const fullContent = article.title\n ? `${article.title}\\n\\n${article.content}`\n : article.content;\n\n // Ensure we have meaningful content (at least 200 characters for full articles)\n if (fullContent.length < 200) {\n throw new Error(\n `Extracted content too short (${fullContent.length} chars) - likely failed to find main content`,\n );\n }\n\n console.log(\n `\u2713 Successfully extracted article: \"${article.title ?? ''}\" (${fullContent.length} chars, ~${Math.round(fullContent.split(' ').length)} words)`,\n );\n\n return fullContent;\n } catch (error) {\n console.warn(\n `Failed to fetch article content from ${url}:`,\n Error.isError(error) ? error.message : error,\n );\n return ''; // Return empty string on failure rather than throwing\n }\n}\nexport function rss(\n feedUrl: string,\n options: {\n maxItems?: number;\n fetchFullArticles?: boolean;\n } = {},\n) {\n const sourceId = `rss:${feedUrl}`;\n const { maxItems = 50, fetchFullArticles = false } = options;\n\n return {\n sourceId,\n instructions: `You answer questions about articles and content from the RSS feed: ${feedUrl}.\n Always cite the article title and link when referencing specific content.\n The feed contains recent articles, blog posts, and news items.\n When referencing content, include the publication date and author when available.\n ${fetchFullArticles ? 'Full article content has been extracted from the original links for comprehensive analysis.' : 'Content includes RSS summaries and descriptions.'}`,\n sources: async function* () {\n const feed = await parseRSSFeed(feedUrl);\n // Add feed summary source\n const feedSummary = `RSS Feed: ${feed.title}\nDescription: ${feed.description}\nWebsite: ${feed.link}\nLanguage: ${feed.language}\nLast Updated: ${feed.lastBuildDate}\nTotal Items: ${feed.items.length}\n\nThis feed provides: ${feed.description}`;\n yield {\n id: 'feed-info',\n content: async () => feedSummary,\n };\n\n // Individual article sources (limit to maxItems)\n const itemsToProcess = feed.items.slice(0, maxItems);\n for (const item of itemsToProcess) {\n const documentId =\n item.guid || item.link || `${item.title}-${item.pubDate}`;\n yield {\n id: documentId,\n content: async () => {\n // Try full article fetch if enabled, fallback to RSS content\n let articleContent = item.contentEncoded || item.description;\n if (fetchFullArticles && item.link) {\n const fullContent = await fetchArticleContent(item.link);\n if (fullContent && fullContent.length > articleContent.length) {\n articleContent = fullContent;\n }\n }\n return `Title: ${item.title}\nAuthor: ${item.author}\nPublished: ${item.pubDate}\nCategories: ${item.categories.join(', ')}\nLink: ${item.link}\n${fetchFullArticles ? 'Full Article Content:' : 'Content:'}\n${articleContent}\n\nSummary: ${item.title} - ${item.description}`;\n },\n };\n }\n },\n };\n}\n", "import fg from 'fast-glob';\nimport { opendir, readFile, stat } from 'node:fs/promises';\nimport { join } from 'node:path';\n\nimport type { Connector } from './connector.ts';\n\nexport function repo(\n dir: string,\n extensions: string[],\n ingestWhen: 'never' | 'contentChanged' | undefined,\n): Connector {\n const sourceId = `repo:${dir}`;\n return {\n sourceId,\n ingestWhen,\n sources: async function* () {\n const paths = await collectFiles(dir, extensions);\n for await (const path of paths) {\n const maxSize = 3 * 1024; // 3KB\n const st = await stat(path);\n if (st.size > maxSize) {\n continue;\n }\n yield {\n id: path,\n metadata: { repo: dir },\n content: () => readFile(path, 'utf8').catch(() => ''),\n };\n }\n },\n };\n}\n\nexport async function* findAllGitRepos(root: string) {\n const stack = [root];\n const skip = [\n 'node_modules',\n 'Library',\n 'Applications',\n 'Pictures',\n 'Movies',\n 'Music',\n 'Downloads',\n '.cache',\n '.npm',\n '.pnpm',\n 'development',\n ];\n while (stack.length) {\n const folder = stack.pop()!;\n const isGitRepo = await stat(join(folder, '.git'))\n .then((st) => st.isDirectory() || st.isFile())\n .catch(() => false);\n if (isGitRepo) {\n yield folder;\n continue;\n }\n const dir = await opendir(folder);\n for await (const dirent of dir) {\n if (\n dirent.isDirectory() &&\n !dirent.isSymbolicLink() &&\n !skip.includes(dirent.name) &&\n !dirent.name.startsWith('.')\n ) {\n stack.push(join(folder, dirent.name));\n }\n }\n }\n}\n\nfunction detectRepoTooling(repo: string) {\n const tools: string[] = [];\n return tools;\n}\n\nasync function gitignore(gitignoreFile: string) {\n const content = await readFile(gitignoreFile, 'utf8').catch(() => '');\n if (!content) {\n return [];\n }\n const patterns = content\n .split('\\n')\n .map((line) => line.trim())\n .filter((line) => line && !line.startsWith('#'));\n return patterns;\n}\n\nexport async function ignorePatterns(repo: string) {\n return [\n // Inherit repo-specific ignore patterns\n ...(await gitignore(join(repo, '.gitignore'))),\n\n // Package managers & dependency dirs\n 'node_modules/**',\n '**/node_modules/**',\n '**/.pnpm/**',\n '**/.npm/**',\n '**/.yarn/**',\n '**/vendor/**', // PHP / Go modules (when vendored)\n '**/3rdparty/**',\n\n // Version control + VCS metadata\n '**/.git/**',\n '**/.svn/**',\n '**/.hg/**',\n\n // OS / system junk\n '**/.DS_Store',\n '**/Thumbs.db',\n '**/Library/**',\n '**/Applications/**',\n '**/Pictures/**',\n '**/Movies/**',\n '**/Music/**',\n '**/Downloads/**',\n '**/.cache/**',\n\n // Environment / secrets (explicit)\n '**/.env',\n '**/.env.*',\n\n // Lockfiles & generated dependency state\n '**/*.lock',\n '**/yarn.lock',\n '**/package-lock.json',\n '**/pnpm-lock.yaml',\n\n // Build / compilation outputs\n '**/dist/**',\n '**/debug/**',\n '**/build/**',\n '**/out/**',\n '**/target/**', // Rust / JVM\n '**/bin/**',\n '**/obj/**',\n '**/classes/**',\n\n // Framework / tool specific build artifacts\n '**/.next/**',\n '**/.vercel/**',\n '**/.turbo/**',\n '**/.docusaurus/**',\n '**/.vite/**',\n '**/.parcel-cache/**',\n '**/.rollup.cache/**',\n '**/.vuepress/**',\n 'cdk.out/**',\n\n // Infra & deployment tooling\n '**/.serverless/**',\n '**/.terraform/**',\n '**/.terragrunt-cache/**',\n '**/.pulumi/**',\n\n // Coverage & testing caches\n '**/coverage/**',\n '**/.nyc_output/**',\n '**/jest-cache/**',\n '**/.pytest_cache/**',\n\n // Language / tooling caches\n '**/__pycache__/**',\n '**/.mypy_cache/**',\n '**/.tox/**',\n '**/.gradle/**',\n '**/.mvn/**',\n '**/.eslintcache',\n '**/.stylelintcache',\n\n // IDE / editor configs + history (we don't want to embed these)\n '**/.idea/**',\n '**/.vscode/**',\n '**/.fleet/**',\n '**/.history/**',\n\n // Virtual environments\n '**/.venv/**',\n '**/venv/**',\n ];\n}\n\nexport async function collectFiles(\n repo: string,\n extensions: string[],\n): Promise<AsyncIterable<string>> {\n const exts = extensions.map((ext) => ext.replace(/^\\./, ''));\n const entries = fg.stream(\n extensions.length > 1 ? `**/*.{${exts.join(',')}}` : `**/*.${exts[0]}`,\n {\n dot: false,\n onlyFiles: true,\n unique: true,\n absolute: true,\n cwd: repo,\n ignore: await ignorePatterns(repo),\n },\n );\n return (async function* () {\n for await (const entry of entries) {\n if (typeof entry !== 'string') {\n throw new TypeError('fast-glob yielded a non-path entry');\n }\n yield entry;\n }\n })();\n}\n"],
|
|
5
|
+
"mappings": ";AAAA,SAAS,aAAa;AA0BtB,SAAS,UAAU,OAAkC;AACnD,SACE,OAAO,UAAU,YACjB,UAAU,QACV,cAAc,SACd,OAAO,MAAM,aAAa,YAC1B,UAAU,UACT,MAAM,SAAS,QAAQ,OAAO,MAAM,SAAS,aAC9C,UAAU,UACT,MAAM,SAAS,QAAQ,OAAO,MAAM,SAAS,aAC9C,gBAAgB,SAChB,OAAO,MAAM,eAAe,YAC5B,kBAAkB,SAClB,OAAO,MAAM,iBAAiB,YAC9B,cAAc,SACd,OAAO,MAAM,aAAa,YAC1B,WAAW,SACX,OAAO,MAAM,UAAU,aACvB,gBAAgB,SAChB,OAAO,MAAM,eAAe;AAEhC;AAEA,SAAS,GAAG,MAAc;AACxB,QAAM,CAAC,OAAOA,OAAM,GAAG,QAAQ,IAAI,KAAK,MAAM,GAAG;AACjD,SAAO;AAAA,IACL,UAAU,YAAY;AACpB,YAAM,MAAM,gCAAgC,KAAK,IAAIA,KAAI,aAAa,SAAS,KAAK,GAAG,CAAC;AACxF,YAAM,MAAM,MAAM,MAAM,GAAG;AAC3B,YAAM,OAAgB,MAAM,IAAI,KAAK;AACrC,UACE,OAAO,SAAS,YAChB,SAAS,QACT,EAAE,aAAa,SACf,OAAO,KAAK,YAAY,UACxB;AACA,cAAM,IAAI,MAAM,uCAAuC,IAAI,EAAE;AAAA,MAC/D;AACA,aAAO,KAAK,KAAK,OAAO;AAAA,IAC1B;AAAA,IACA,SAAS,OAAOA,OAAc,OAA4B,CAAC,MAAM;AAC/D,YAAM,CAACC,QAAO,QAAQ,IAAID,MAAK,MAAM,GAAG;AACxC,YAAM,UAAU;AAChB,YAAM,WAAW;AACjB,YAAM,WAAsB,CAAC;AAC7B,UAAI,OAAO;AAEX,eAAS,OAAO,GAAG,QAAQ,YAAY,CAAC,MAAM,QAAQ;AACpD,cAAM,MAAM,gCAAgCC,MAAK,IAAI,QAAQ,sBAAsB,OAAO,SAAS,IAAI;AACvG,cAAM,MAAM,MAAM,MAAM,KAAK;AAAA,UAC3B,SAAS,EAAE,QAAQ,8BAA8B;AAAA,QACnD,CAAC;AACD,YAAI,CAAC,IAAI,GAAI;AACb,cAAM,WAAoB,MAAM,IAAI,KAAK;AACzC,YAAI,CAAC,MAAM,QAAQ,QAAQ,KAAK,CAAC,SAAS,MAAM,SAAS,GAAG;AAC1D,gBAAM,IAAI,MAAM,2CAA2CD,KAAI,EAAE;AAAA,QACnE;AACA,YAAI,CAAC,SAAS,OAAQ;AACtB,mBAAW,OAAO,UAAU;AAC1B,cAAI,CAAC,KAAK,iBAAiB,IAAI,MAAO;AACtC,cAAI,CAAC,KAAK,qBAAqB,IAAI,WAAY;AAE/C,mBAAS,KAAK,GAAG;AACjB,cAAI,KAAK,YAAY,IAAI,aAAa,KAAK,UAAU;AACnD,gBAAI,KAAK,cAAc,OAAO;AAC5B,uBAAS,IAAI;AAAA,YACf;AACA,mBAAO;AACP;AAAA,UACF;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAEO,IAAM,SAAS;AAAA,EACpB,KAAK,UAA6B;AAChC,UAAM,WAAW,eAAe,QAAQ;AACxC,UAAM,aAAa;AACnB,WAAO;AAAA,MACL;AAAA,MACA,SAAS,mBAAmB;AAC1B,cAAM,EAAE,UAAAE,UAAS,IAAI,GAAG,QAAQ;AAChC,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,MAAMA,UAAS;AAAA,QAC1B;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,SAAS,CAACF,OAAc,UAA+B,CAAC,MAAiB;AACvE,UAAM,WAAW,mBAAmBA,KAAI;AACxC,WAAO;AAAA,MACL;AAAA,MACA,SAAS,mBAAmB;AAC1B,cAAM,WAAW,MAAM,GAAGA,KAAI,EAAE,QAAQA,OAAM,OAAO;AACrD,mBAAW,OAAO,UAAU;AAC1B,gBAAM;AAAA,YACJ,IAAI,GAAGA,KAAI,YAAY,IAAI,QAAQ;AAAA,YACnC,SAAS,YAAY;AACnB,oBAAM,OAAO,IAAI,QAAQ,IAAI;AAC7B,oBAAM,OAAO,IAAI,QAAQ;AACzB,qBAAO,YAAY,IAAI;AAAA,OAAU,IAAI,QAAQ;AAAA,gBAAmB,IAAI,YAAY;AAAA,cAAiB,IAAI,UAAU;AAAA,OAAU,IAAI,QAAQ;AAAA,SAAY,IAAI,KAAK;AAAA,cAAiB,IAAI,UAAU;AAAA;AAAA,EAAO,IAAI;AAAA,YACtM;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,CACJ,SACA,SASc;AACd,UAAM,WAAW,eAAe,OAAO;AACvC,UAAM;AAAA,MACJ,WAAW;AAAA,QACT;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA;AAAA,MACA,oBAAoB;AAAA,MACpB,oBAAoB;AAAA,MACpB;AAAA,MACA;AAAA,MACA;AAAA,IACF,IAAI;AAEJ,mBAAe,kBAAmC;AAChD,YAAM,OAAO;AAAA,QACX;AAAA,QACA,GAAI,SAAS,CAAC,MAAM,MAAM,IAAI,CAAC;AAAA,QAC/B,GAAG,SAAS,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAAA,QACpC,GAAG,SAAS,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAAA,QACpC,GAAI,oBAAoB,CAAC,sBAAsB,IAAI,CAAC;AAAA,QACpD,GAAI,oBAAoB,CAAC,sBAAsB,IAAI,CAAC;AAAA,QACpD;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,YAAM,EAAE,OAAO,IAAI,MAAM,MAAM,OAAO,MAAM;AAAA,QAC1C,KAAK,cACD,EAAE,GAAG,QAAQ,KAAK,cAAc,YAAY,IAC5C,QAAQ;AAAA,QACZ,QAAQ;AAAA,QACR,QAAQ;AAAA,MACV,CAAC;AACD,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,SAAS,mBAAmB;AAC1B,cAAM,UAAU,MAAM,gBAAgB;AACtC,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AAAA,QACvB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC3NA,SAAS,oBAAoB;AAEtB,SAAS,OAAO,QAAgB;AACrC,QAAM,eAAe,IAAI,aAAa,EAAE,OAAO,CAAC;AAChD,QAAM,WAAW;AACjB,SAAO;AAAA,IACL;AAAA,IACA,cACE;AAAA,IACF,SAAS,YAAY;AACnB,YAAM,OAAO,MAAM,aAAa;AAChC,YAAM,SAAS,MAAM,KAAK,eAAe;AACzC,aAAO,QAAQ;AAAA,QACb,OAAO,MAAM,IAAI,OAAO,OAAO;AAC7B,gBAAM,QAAQ,MAAM,GAAG;AACvB,iBAAO;AAAA,YACL,IAAI,GAAG;AAAA,YACP,SAAS,YACP,UAAU,GAAG,KAAK;AAAA,eAAkB,GAAG,eAAe,gBAAgB;AAAA,UAAa,OAAO,QAAQ,SAAS;AAAA,MAAS,GAAG,EAAE;AAAA,UAC7H;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;;;ACxBA,OAAO,QAAQ;AACf,OAAO,YAAY;AACnB,SAAS,UAAU,YAAY;AAC/B,SAAS,SAAS,MAAM,UAAU,eAAe;AAOjD,IAAM,iBAAiB,oBAAI,IAAuC;AAKlE,eAAe,yBACb,YACoC;AACpC,QAAM,eAAe,QAAQ,UAAU;AAEvC,MAAI,eAAe,IAAI,YAAY,GAAG;AACpC,WAAO,eAAe,IAAI,YAAY;AAAA,EACxC;AAEA,QAAM,KAAK,OAAO;AAGlB,QAAM,eAAe,aAAa,MAAM,GAAG;AAC3C,QAAM,iBAA2B,CAAC;AAGlC,WAAS,IAAI,GAAG,KAAK,aAAa,QAAQ,KAAK;AAC7C,UAAM,cAAc,aAAa,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG;AACrD,UAAM,gBAAgB,KAAK,aAAa,YAAY;AAEpD,QAAI;AACF,YAAM,KAAK,aAAa;AACxB,qBAAe,KAAK,aAAa;AAAA,IACnC,QAAQ;AAAA,IAER;AAAA,EACF;AAGA,aAAW,iBAAiB,gBAAgB;AAC1C,QAAI;AACF,YAAM,UAAU,MAAM,SAAS,eAAe,MAAM;AACpD,YAAM,WAAW,QACd,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,QAAQ,CAAC,KAAK,WAAW,GAAG,CAAC;AAEjD,SAAG,IAAI,QAAQ;AAAA,IACjB,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,iBAAe,IAAI,cAAc,EAAE;AACnC,SAAO;AACT;AAKA,eAAe,iBACb,KACA,SACmB;AAEnB,QAAM,WAAW,MAAM,GAAG,SAAS;AAAA,IACjC,KAAK;AAAA,IACL,UAAU;AAAA,IACV,WAAW;AAAA,IACX;AAAA,IACA,qBAAqB;AAAA,IACrB,QAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF,CAAC;AAED,MAAI,SAAS,WAAW,GAAG;AACzB,WAAO,CAAC;AAAA,EACV;AAGA,QAAM,aAAa,oBAAI,IAAsB;AAE7C,aAAW,QAAQ,UAAU;AAC3B,UAAM,MAAM,QAAQ,IAAI;AACxB,QAAI,CAAC,WAAW,IAAI,GAAG,GAAG;AACxB,iBAAW,IAAI,KAAK,CAAC,CAAC;AAAA,IACxB;AACA,eAAW,IAAI,GAAG,EAAG,KAAK,IAAI;AAAA,EAChC;AAEA,QAAM,gBAA0B,CAAC;AAGjC,aAAW,CAAC,KAAK,KAAK,KAAK,YAAY;AACrC,UAAM,eAAe,MAAM,yBAAyB,GAAG;AAEvD,eAAW,QAAQ,OAAO;AACxB,UAAI,CAAC,aAAa,QAAQ,SAAS,KAAK,IAAI,CAAC,GAAG;AAC9C,sBAAc,KAAK,IAAI;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAEA,SAAO,cAAc,KAAK;AAC5B;AAEO,SAAS,MACd,SACA,UAA2C,CAAC,GACjC;AACX,QAAM,EAAE,KAAK,GAAG,OAAO,IAAI;AAC3B,QAAM,WAAW,QAAQ,OAAO;AAChC,SAAO;AAAA,IACL,GAAG;AAAA,IACH;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM,iBAAiB,OAAO,QAAQ,IAAI,GAAG,OAAO;AAClE,iBAAW,YAAY,OAAO;AAC5B,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,MAAM,SAAS,UAAU,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,QAC1D;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC3IA,SAAS,MAAM,YAAAG,iBAAgB;AAC/B,SAAS,aAAa,wBAAwB;AAIvC,SAAS,IAAI,SAA4B;AAC9C,QAAM,WAAW,OAAO,OAAO;AAE/B,SAAO;AAAA,IACL;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM,MAAM;AAAA,QACxB,KAAK,SAAS,EAAE,SAAS,CAAC,sBAAsB,YAAY,EAAE,CAAC;AAAA,MACjE;AACA,iBAAW,QAAQ,OAAO;AACxB,YAAI,CAAC,KAAK,YAAY,EAAE,SAAS,MAAM,EAAG;AAC1C,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AACnB,kBAAM,SAAS,MAAMA,UAAS,IAAI;AAClC,kBAAMC,OAAM,MAAM,iBAAiB,IAAI,WAAW,MAAM,CAAC;AACzD,kBAAM,EAAE,KAAK,IAAI,MAAM,YAAYA,MAAK,EAAE,YAAY,KAAK,CAAC;AAC5D,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,QAAQ,QAA2B;AACjD,QAAM,QAAQ,eAAe,KAAK,MAAM;AACxC,QAAM,WAAW,OAAO,QAAQ,QAAQ,MAAM,IAAI,MAAM;AAExD,SAAO;AAAA,IACL;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM;AAAA,QACJ,IAAI;AAAA,QACJ,SAAS,YAAY;AACnB,cAAI;AAEJ,cAAI,OAAO;AACT,kBAAM,WAAW,MAAM,MAAM,MAAM;AACnC,gBAAI,CAAC,SAAS,IAAI;AAChB,oBAAM,IAAI;AAAA,gBACR,QAAQ,SAAS,MAAM,KAAK,SAAS,UAAU;AAAA,cACjD;AAAA,YACF;AACA,qBAAS,IAAI,WAAW,MAAM,SAAS,YAAY,CAAC;AAAA,UACtD,OAAO;AACL,kBAAM,aAAa,MAAMD,UAAS,MAAM;AACxC,qBAAS,IAAI,WAAW,WAAW,MAAM;AAAA,UAC3C;AAEA,gBAAMC,OAAM,MAAM,iBAAiB,IAAI,WAAW,MAAM,CAAC;AACzD,gBAAM,EAAE,KAAK,IAAI,MAAM,YAAYA,MAAK,EAAE,YAAY,KAAK,CAAC;AAC5D,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;AC9DA,SAAS,uCAAuC;AAChD,OAAO,YAAY;AACnB,OAAO,YAAY;AAEnB,IAAM,WAAW,gCAAgC;AAAA;AAAA,EAE/C,WAAW,OAAO;AACpB,CAAC;AAmBD,IAAM,YAAY,IAAI,OAA+B;AAAA,EACnD,cAAc;AAAA,IACZ,MAAM,CAAC,aAAa,YAAY,kBAAkB,WAAW;AAAA,IAC7D,MAAM,CAAC,YAAY,WAAW,aAAa,MAAM;AAAA,EACnD;AACF,CAAC;AAED,eAAe,aAAa,SAAiB;AAC3C,MAAI;AACF,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,UAAM,OAAO,MAAM,UAAU,SAAS,OAAO;AAE7C,WAAO;AAAA,MACL,OAAO,KAAK,SAAS;AAAA,MACrB,aAAa,KAAK,eAAe;AAAA,MACjC,MAAM,KAAK,QAAQ;AAAA,MACnB,UAAU,KAAK,YAAY;AAAA,MAC3B,eAAe,KAAK,kBAAiB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC5D,OAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,QAC/B,OAAO,KAAK,SAAS;AAAA,QACrB,aAAa,KAAK,WAAW,KAAK,WAAW,KAAK,kBAAkB;AAAA,QACpE,MAAM,KAAK,QAAQ;AAAA,QACnB,SAAS,KAAK,WAAW,KAAK,WAAW;AAAA,QACzC,QAAQ,KAAK,WAAW,KAAK,UAAU;AAAA,QACvC,YAAY,MAAM,QAAQ,KAAK,UAAU,IACrC,KAAK,aACL,KAAK,WACH,CAAC,KAAK,QAAQ,IACd,CAAC;AAAA,QACP,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,QAChC,gBAAgB,KAAK,iBAAiB,KAAK,KAAK,WAAW;AAAA,MAC7D,EAAE;AAAA,IACJ;AAAA,EACF,SAAS,OAAO;AACd,YAAQ,MAAM,4BAA4B,OAAO,KAAK,KAAK;AAC3D,UAAM,IAAI;AAAA,MACR,uBAAuB,MAAM,QAAQ,KAAK,IAAI,MAAM,UAAU,eAAe;AAAA,MAC7E,EAAE,OAAO,MAAM;AAAA,IACjB;AAAA,EACF;AACF;AAEA,eAAe,oBAAoB,KAA8B;AAC/D,MAAI;AACF,YAAQ,IAAI,uCAAuC,GAAG,EAAE;AAExD,UAAM,UAAU,MAAM,SAAS;AAAA,MAC7B,EAAE,IAAI;AAAA,MACN,EAAE,YAAY,KAAK,UAAU,CAAC,GAAG,SAAS,CAAC,EAAE;AAAA,IAC/C;AACA,WAAO,GAAG,UAAU,SAAS,yCAAyC;AACtE,QAAI,QAAQ,SAAS,SAAS;AAC5B,YAAM,IAAI,MAAM,QAAQ,OAAO;AAAA,IACjC;AAGA,UAAM,cAAc,QAAQ,QACxB,GAAG,QAAQ,KAAK;AAAA;AAAA,EAAO,QAAQ,OAAO,KACtC,QAAQ;AAGZ,QAAI,YAAY,SAAS,KAAK;AAC5B,YAAM,IAAI;AAAA,QACR,gCAAgC,YAAY,MAAM;AAAA,MACpD;AAAA,IACF;AAEA,YAAQ;AAAA,MACN,2CAAsC,QAAQ,SAAS,EAAE,MAAM,YAAY,MAAM,YAAY,KAAK,MAAM,YAAY,MAAM,GAAG,EAAE,MAAM,CAAC;AAAA,IACxI;AAEA,WAAO;AAAA,EACT,SAAS,OAAO;AACd,YAAQ;AAAA,MACN,wCAAwC,GAAG;AAAA,MAC3C,MAAM,QAAQ,KAAK,IAAI,MAAM,UAAU;AAAA,IACzC;AACA,WAAO;AAAA,EACT;AACF;AACO,SAAS,IACd,SACA,UAGI,CAAC,GACL;AACA,QAAM,WAAW,OAAO,OAAO;AAC/B,QAAM,EAAE,WAAW,IAAI,oBAAoB,MAAM,IAAI;AAErD,SAAO;AAAA,IACL;AAAA,IACA,cAAc,sEAAsE,OAAO;AAAA;AAAA;AAAA;AAAA,QAIvF,oBAAoB,gGAAgG,kDAAkD;AAAA,IAC1K,SAAS,mBAAmB;AAC1B,YAAM,OAAO,MAAM,aAAa,OAAO;AAEvC,YAAM,cAAc,aAAa,KAAK,KAAK;AAAA,eAClC,KAAK,WAAW;AAAA,WACpB,KAAK,IAAI;AAAA,YACR,KAAK,QAAQ;AAAA,gBACT,KAAK,aAAa;AAAA,eACnB,KAAK,MAAM,MAAM;AAAA;AAAA,sBAEV,KAAK,WAAW;AAChC,YAAM;AAAA,QACJ,IAAI;AAAA,QACJ,SAAS,YAAY;AAAA,MACvB;AAGA,YAAM,iBAAiB,KAAK,MAAM,MAAM,GAAG,QAAQ;AACnD,iBAAW,QAAQ,gBAAgB;AACjC,cAAM,aACJ,KAAK,QAAQ,KAAK,QAAQ,GAAG,KAAK,KAAK,IAAI,KAAK,OAAO;AACzD,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AAEnB,gBAAI,iBAAiB,KAAK,kBAAkB,KAAK;AACjD,gBAAI,qBAAqB,KAAK,MAAM;AAClC,oBAAM,cAAc,MAAM,oBAAoB,KAAK,IAAI;AACvD,kBAAI,eAAe,YAAY,SAAS,eAAe,QAAQ;AAC7D,iCAAiB;AAAA,cACnB;AAAA,YACF;AACA,mBAAO,UAAU,KAAK,KAAK;AAAA,UAC7B,KAAK,MAAM;AAAA,aACR,KAAK,OAAO;AAAA,cACX,KAAK,WAAW,KAAK,IAAI,CAAC;AAAA,QAChC,KAAK,IAAI;AAAA,EACf,oBAAoB,0BAA0B,UAAU;AAAA,EACxD,cAAc;AAAA;AAAA,WAEL,KAAK,KAAK,MAAM,KAAK,WAAW;AAAA,UACjC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;;;ACzKA,OAAOC,SAAQ;AACf,SAAS,SAAS,YAAAC,WAAU,QAAAC,aAAY;AACxC,SAAS,QAAAC,aAAY;AAId,SAAS,KACd,KACA,YACA,YACW;AACX,QAAM,WAAW,QAAQ,GAAG;AAC5B,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM,aAAa,KAAK,UAAU;AAChD,uBAAiB,QAAQ,OAAO;AAC9B,cAAM,UAAU,IAAI;AACpB,cAAM,KAAK,MAAMD,MAAK,IAAI;AAC1B,YAAI,GAAG,OAAO,SAAS;AACrB;AAAA,QACF;AACA,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,UAAU,EAAE,MAAM,IAAI;AAAA,UACtB,SAAS,MAAMD,UAAS,MAAM,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,QACtD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,gBAAuB,gBAAgB,MAAc;AACnD,QAAM,QAAQ,CAAC,IAAI;AACnB,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO,MAAM,QAAQ;AACnB,UAAM,SAAS,MAAM,IAAI;AACzB,UAAM,YAAY,MAAMC,MAAKC,MAAK,QAAQ,MAAM,CAAC,EAC9C,KAAK,CAAC,OAAO,GAAG,YAAY,KAAK,GAAG,OAAO,CAAC,EAC5C,MAAM,MAAM,KAAK;AACpB,QAAI,WAAW;AACb,YAAM;AACN;AAAA,IACF;AACA,UAAM,MAAM,MAAM,QAAQ,MAAM;AAChC,qBAAiB,UAAU,KAAK;AAC9B,UACE,OAAO,YAAY,KACnB,CAAC,OAAO,eAAe,KACvB,CAAC,KAAK,SAAS,OAAO,IAAI,KAC1B,CAAC,OAAO,KAAK,WAAW,GAAG,GAC3B;AACA,cAAM,KAAKA,MAAK,QAAQ,OAAO,IAAI,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AACF;AAOA,eAAe,UAAU,eAAuB;AAC9C,QAAM,UAAU,MAAMC,UAAS,eAAe,MAAM,EAAE,MAAM,MAAM,EAAE;AACpE,MAAI,CAAC,SAAS;AACZ,WAAO,CAAC;AAAA,EACV;AACA,QAAM,WAAW,QACd,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,QAAQ,CAAC,KAAK,WAAW,GAAG,CAAC;AACjD,SAAO;AACT;AAEA,eAAsB,eAAeC,OAAc;AACjD,SAAO;AAAA;AAAA,IAEL,GAAI,MAAM,UAAUC,MAAKD,OAAM,YAAY,CAAC;AAAA;AAAA,IAG5C;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,EACF;AACF;AAEA,eAAsB,aACpBA,OACA,YACgC;AAChC,QAAM,OAAO,WAAW,IAAI,CAAC,QAAQ,IAAI,QAAQ,OAAO,EAAE,CAAC;AAC3D,QAAM,UAAUE,IAAG;AAAA,IACjB,WAAW,SAAS,IAAI,SAAS,KAAK,KAAK,GAAG,CAAC,MAAM,QAAQ,KAAK,CAAC,CAAC;AAAA,IACpE;AAAA,MACE,KAAK;AAAA,MACL,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,KAAKF;AAAA,MACL,QAAQ,MAAM,eAAeA,KAAI;AAAA,IACnC;AAAA,EACF;AACA,UAAQ,mBAAmB;AACzB,qBAAiB,SAAS,SAAS;AACjC,UAAI,OAAO,UAAU,UAAU;AAC7B,cAAM,IAAI,UAAU,oCAAoC;AAAA,MAC1D;AACA,YAAM;AAAA,IACR;AAAA,EACF,GAAG;AACL;",
|
|
6
6
|
"names": ["repo", "owner", "readFile", "readFile", "pdf", "fg", "readFile", "stat", "join", "readFile", "repo", "join", "fg"]
|
|
7
7
|
}
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import type { Connector } from './connector.ts';
|
|
2
|
-
export declare function local(pattern: string, options?: {
|
|
3
|
-
ingestWhen?: 'never' | 'contentChanged' | 'expired';
|
|
4
|
-
expiresAfter?: number;
|
|
1
|
+
import type { Connector, IngestPolicy } from './connector.ts';
|
|
2
|
+
export declare function local(pattern: string, options?: IngestPolicy & {
|
|
5
3
|
cwd?: string;
|
|
6
4
|
}): Connector;
|
|
7
5
|
//# sourceMappingURL=local.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/local.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/local.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAmH9D,wBAAgB,KAAK,CACnB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,YAAY,GAAG;IAAE,GAAG,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5C,SAAS,CAgBX"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Connector } from './connector.ts';
|
|
2
|
-
export declare function repo(dir: string, extensions: string[], ingestWhen:
|
|
2
|
+
export declare function repo(dir: string, extensions: string[], ingestWhen: 'never' | 'contentChanged' | undefined): Connector;
|
|
3
3
|
export declare function findAllGitRepos(root: string): AsyncGenerator<string, void, unknown>;
|
|
4
4
|
export declare function ignorePatterns(repo: string): Promise<string[]>;
|
|
5
5
|
export declare function collectFiles(repo: string, extensions: string[]): Promise<AsyncIterable<string>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/repo.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,wBAAgB,IAAI,CAClB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,EAAE,EACpB,UAAU,EAAE,
|
|
1
|
+
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/repo.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,wBAAgB,IAAI,CAClB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,EAAE,EACpB,UAAU,EAAE,OAAO,GAAG,gBAAgB,GAAG,SAAS,GACjD,SAAS,CAqBX;AAED,wBAAuB,eAAe,CAAC,IAAI,EAAE,MAAM,yCAoClD;AAmBD,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,qBA4FhD;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,EAAE,GACnB,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAqBhC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rss.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/rss.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"rss.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/rss.ts"],"names":[],"mappings":"AA0GA,wBAAgB,GAAG,CACjB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CACxB;;;;;;;EA0DP"}
|
package/dist/lib/ingest.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/lib/ingest.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAEzD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,KAAK,EAAE,KAAK,CAAC;IACb,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED,wBAAsB,MAAM,CAC1B,MAAM,EAAE,eAAe,EACvB,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,
|
|
1
|
+
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/lib/ingest.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAEzD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,KAAK,EAAE,KAAK,CAAC;IACb,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED,wBAAsB,MAAM,CAC1B,MAAM,EAAE,eAAe,EACvB,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,iBAuChC;AAaD,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AA+EF,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAQnC;AAED,wBAAsB,eAAe,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,qBAGhE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"similiarty-search.d.ts","sourceRoot":"","sources":["../../src/lib/similiarty-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,eAAe,EAAU,MAAM,aAAa,CAAC;AAE3D,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,UAAU,CAAC;;;;;;
|
|
1
|
+
{"version":3,"file":"similiarty-search.d.ts","sourceRoot":"","sources":["../../src/lib/similiarty-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,eAAe,EAAU,MAAM,aAAa,CAAC;AAE3D,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,UAAU,CAAC;;;;;;KA+C1C"}
|
|
@@ -19,7 +19,7 @@ export declare class SQLiteStore implements Store {
|
|
|
19
19
|
documentId: string;
|
|
20
20
|
}): import("node:sqlite").StatementResultingChanges;
|
|
21
21
|
sourceExists(sourceId: string): boolean;
|
|
22
|
-
sourceExpired(sourceId: string): boolean;
|
|
22
|
+
sourceExpired(sourceId: string, now: Date): boolean;
|
|
23
23
|
setSourceExpiry(sourceId: string, expiryDate: Date): void;
|
|
24
24
|
index(sourceId: string, corpus: Corpus, expiryDate?: Date): Promise<void>;
|
|
25
25
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,KAAK,EACV,KAAK,EACL,MAAM,EACN,QAAQ,EACR,aAAa,EACb,YAAY,EACZ,KAAK,EACN,MAAM,aAAa,CAAC;AAQrB,qBAAa,WAAY,YAAW,KAAK;;gBAE3B,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM;IA8DzC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ;IAyCtE,SAAS,CAAC,MAAM,EAAE;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IAiBD,SAAS,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,IAKhD,OAAO,KAAK;IAStB,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAMvD,YAAY,CAAC,QAAQ,EAAE,MAAM;IAO7B,aAAa,CAAC,QAAQ,EAAE,MAAM;
|
|
1
|
+
{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,KAAK,EACV,KAAK,EACL,MAAM,EACN,QAAQ,EACR,aAAa,EACb,YAAY,EACZ,KAAK,EACN,MAAM,aAAa,CAAC;AAQrB,qBAAa,WAAY,YAAW,KAAK;;gBAE3B,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM;IA8DzC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ;IAyCtE,SAAS,CAAC,MAAM,EAAE;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IAiBD,SAAS,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,IAKhD,OAAO,KAAK;IAStB,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAMvD,YAAY,CAAC,QAAQ,EAAE,MAAM;IAO7B,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI;IAWzC,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI;IAQ5C,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI;CAuDhE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,YAAY,GAAG,MAAM,CASpE"}
|
|
@@ -27,7 +27,7 @@ export interface SearchResult {
|
|
|
27
27
|
export interface Store {
|
|
28
28
|
search: (query: string, options: SearchOptions, embedder: Embedder) => Promise<SearchResult[]>;
|
|
29
29
|
sourceExists: (sourceId: string) => Promise<boolean> | boolean;
|
|
30
|
-
sourceExpired: (sourceId: string) => Promise<boolean> | boolean;
|
|
30
|
+
sourceExpired: (sourceId: string, now: Date) => Promise<boolean> | boolean;
|
|
31
31
|
setSourceExpiry: (sourceId: string, expiryDate: Date) => Promise<void> | void;
|
|
32
32
|
index: (sourceId: string, corpus: Corpus, expiryDate?: Date) => Promise<void>;
|
|
33
33
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../../src/lib/stores/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpC,MAAM,MAAM,KAAK,GAAG;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,SAAS,GAAG,YAAY,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC;IACtD,UAAU,EAAE,CAAC,SAAS,GAAG,YAAY,CAAC,EAAE,CAAC;IACzC,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAAC;AAEH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IAEjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,cAAc,CAAC,KAAK,CAAC,CAAC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,CAAC;CAC5C,CAAC;AAEF,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC1C;AAED,MAAM,WAAW,KAAK;IACpB,MAAM,EAAE,CACN,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,aAAa,EACtB,QAAQ,EAAE,QAAQ,KACf,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IAC7B,YAAY,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAC/D,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../../src/lib/stores/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpC,MAAM,MAAM,KAAK,GAAG;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,SAAS,GAAG,YAAY,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC;IACtD,UAAU,EAAE,CAAC,SAAS,GAAG,YAAY,CAAC,EAAE,CAAC;IACzC,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAAC;AAEH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IAEjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,cAAc,CAAC,KAAK,CAAC,CAAC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,CAAC;CAC5C,CAAC;AAEF,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC1C;AAED,MAAM,WAAW,KAAK;IACpB,MAAM,EAAE,CACN,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,aAAa,EACtB,QAAQ,EAAE,QAAQ,KACf,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IAC7B,YAAY,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAC/D,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAC3E,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC9E,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/E"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@deepagents/retrieval",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.7.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -40,22 +40,26 @@
|
|
|
40
40
|
"access": "public"
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
|
-
"@ai-sdk/tool-web-fetch": "^1.0.
|
|
43
|
+
"@ai-sdk/tool-web-fetch": "^1.0.16",
|
|
44
|
+
"@huggingface/transformers": "^4.2.0",
|
|
45
|
+
"@langchain/core": "^1.2.3",
|
|
44
46
|
"@langchain/textsplitters": "^1.0.1",
|
|
45
|
-
"@zukhruf/sqlite-drift": "^0.8.10",
|
|
46
|
-
"@zukhruf/sqlite-open": "^0.8.10",
|
|
47
|
-
"@zukhruf/sqlite-transaction": "^0.8.10",
|
|
48
|
-
"execa": "^10.0.0",
|
|
49
47
|
"@linear/sdk": "^88.2.0",
|
|
48
|
+
"@zukhruf/sqlite-drift": "^0.8.30",
|
|
49
|
+
"@zukhruf/sqlite-open": "^0.8.30",
|
|
50
|
+
"@zukhruf/sqlite-transaction": "^0.8.30",
|
|
51
|
+
"ai": "^7.0.166",
|
|
52
|
+
"dedent": "1.7.2",
|
|
53
|
+
"execa": "^10.0.0",
|
|
50
54
|
"fast-glob": "^3.3.3",
|
|
51
55
|
"ignore": "^7.0.6",
|
|
52
|
-
"unpdf": "^1.6.2",
|
|
53
|
-
"rss-parser": "^3.13.0",
|
|
54
|
-
"@huggingface/transformers": "^4.2.0",
|
|
55
56
|
"langchain": "^1.5.3",
|
|
56
|
-
"
|
|
57
|
+
"rss-parser": "^3.13.0",
|
|
57
58
|
"sqlite-vec": "^0.1.9",
|
|
58
|
-
"
|
|
59
|
-
|
|
59
|
+
"unpdf": "^1.6.2"
|
|
60
|
+
},
|
|
61
|
+
"devDependencies": {
|
|
62
|
+
"@sinonjs/fake-timers": "^15.4.0",
|
|
63
|
+
"@zukhruf/fs": "^0.8.30"
|
|
60
64
|
}
|
|
61
65
|
}
|