@contractspec/lib.knowledge 1.56.1 → 1.58.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/access/guard.d.ts +13 -17
  2. package/dist/access/guard.d.ts.map +1 -1
  3. package/dist/access/guard.js +60 -49
  4. package/dist/access/index.d.ts +2 -2
  5. package/dist/access/index.d.ts.map +1 -0
  6. package/dist/access/index.js +60 -2
  7. package/dist/index.d.ts +6 -12
  8. package/dist/index.d.ts.map +1 -0
  9. package/dist/index.js +455 -12
  10. package/dist/ingestion/document-processor.d.ts +18 -20
  11. package/dist/ingestion/document-processor.d.ts.map +1 -1
  12. package/dist/ingestion/document-processor.js +63 -53
  13. package/dist/ingestion/embedding-service.d.ts +7 -11
  14. package/dist/ingestion/embedding-service.d.ts.map +1 -1
  15. package/dist/ingestion/embedding-service.js +26 -25
  16. package/dist/ingestion/gmail-adapter.d.ts +13 -17
  17. package/dist/ingestion/gmail-adapter.d.ts.map +1 -1
  18. package/dist/ingestion/gmail-adapter.js +67 -46
  19. package/dist/ingestion/index.d.ts +6 -6
  20. package/dist/ingestion/index.d.ts.map +1 -0
  21. package/dist/ingestion/index.js +221 -6
  22. package/dist/ingestion/storage-adapter.d.ts +10 -14
  23. package/dist/ingestion/storage-adapter.d.ts.map +1 -1
  24. package/dist/ingestion/storage-adapter.js +31 -26
  25. package/dist/ingestion/vector-indexer.d.ts +11 -15
  26. package/dist/ingestion/vector-indexer.d.ts.map +1 -1
  27. package/dist/ingestion/vector-indexer.js +32 -32
  28. package/dist/node/access/guard.js +60 -0
  29. package/dist/node/access/index.js +60 -0
  30. package/dist/node/index.js +454 -0
  31. package/dist/node/ingestion/document-processor.js +64 -0
  32. package/dist/node/ingestion/embedding-service.js +26 -0
  33. package/dist/node/ingestion/gmail-adapter.js +72 -0
  34. package/dist/node/ingestion/index.js +221 -0
  35. package/dist/node/ingestion/storage-adapter.js +31 -0
  36. package/dist/node/ingestion/vector-indexer.js +32 -0
  37. package/dist/node/query/index.js +79 -0
  38. package/dist/node/query/service.js +79 -0
  39. package/dist/node/retriever/index.js +100 -0
  40. package/dist/node/retriever/interface.js +0 -0
  41. package/dist/node/retriever/static-retriever.js +43 -0
  42. package/dist/node/retriever/vector-retriever.js +58 -0
  43. package/dist/node/types.js +0 -0
  44. package/dist/query/index.d.ts +2 -2
  45. package/dist/query/index.d.ts.map +1 -0
  46. package/dist/query/index.js +79 -2
  47. package/dist/query/service.d.ts +20 -24
  48. package/dist/query/service.d.ts.map +1 -1
  49. package/dist/query/service.js +76 -62
  50. package/dist/retriever/index.d.ts +4 -4
  51. package/dist/retriever/index.d.ts.map +1 -0
  52. package/dist/retriever/index.js +100 -3
  53. package/dist/retriever/interface.d.ts +38 -43
  54. package/dist/retriever/interface.d.ts.map +1 -1
  55. package/dist/retriever/interface.js +1 -0
  56. package/dist/retriever/static-retriever.d.ts +13 -18
  57. package/dist/retriever/static-retriever.d.ts.map +1 -1
  58. package/dist/retriever/static-retriever.js +42 -46
  59. package/dist/retriever/vector-retriever.d.ts +23 -28
  60. package/dist/retriever/vector-retriever.d.ts.map +1 -1
  61. package/dist/retriever/vector-retriever.js +57 -59
  62. package/dist/types.d.ts +34 -39
  63. package/dist/types.d.ts.map +1 -1
  64. package/dist/types.js +1 -0
  65. package/package.json +152 -45
  66. package/dist/access/guard.js.map +0 -1
  67. package/dist/ingestion/document-processor.js.map +0 -1
  68. package/dist/ingestion/embedding-service.js.map +0 -1
  69. package/dist/ingestion/gmail-adapter.js.map +0 -1
  70. package/dist/ingestion/storage-adapter.js.map +0 -1
  71. package/dist/ingestion/vector-indexer.js.map +0 -1
  72. package/dist/query/service.js.map +0 -1
  73. package/dist/retriever/static-retriever.js.map +0 -1
  74. package/dist/retriever/vector-retriever.js.map +0 -1
package/dist/index.js CHANGED
@@ -1,12 +1,455 @@
1
- import { StaticRetriever, createStaticRetriever } from "./retriever/static-retriever.js";
2
- import { VectorRetriever, createVectorRetriever } from "./retriever/vector-retriever.js";
3
- import { KnowledgeQueryService } from "./query/service.js";
4
- import { DocumentProcessor } from "./ingestion/document-processor.js";
5
- import { EmbeddingService } from "./ingestion/embedding-service.js";
6
- import { VectorIndexer } from "./ingestion/vector-indexer.js";
7
- import { GmailIngestionAdapter } from "./ingestion/gmail-adapter.js";
8
- import { StorageIngestionAdapter } from "./ingestion/storage-adapter.js";
9
- import "./ingestion/index.js";
10
- import { KnowledgeAccessGuard } from "./access/guard.js";
11
-
12
- export { DocumentProcessor, EmbeddingService, GmailIngestionAdapter, KnowledgeAccessGuard, KnowledgeQueryService, StaticRetriever, StorageIngestionAdapter, VectorIndexer, VectorRetriever, createStaticRetriever, createVectorRetriever };
1
+ // @bun
2
+ // src/access/guard.ts
3
+ var DEFAULT_DISALLOWED_WRITE = ["external", "ephemeral"];
4
+
5
+ class KnowledgeAccessGuard {
6
+ disallowedWrite;
7
+ requireWorkflowBinding;
8
+ requireAgentBinding;
9
+ constructor(options = {}) {
10
+ this.disallowedWrite = new Set(options.disallowWriteCategories ?? DEFAULT_DISALLOWED_WRITE);
11
+ this.requireWorkflowBinding = options.requireWorkflowBinding ?? true;
12
+ this.requireAgentBinding = options.requireAgentBinding ?? false;
13
+ }
14
+ checkAccess(spaceBinding, context, appConfig) {
15
+ const { binding, space } = spaceBinding;
16
+ if (binding.required !== false && !this.isSpaceBound(spaceBinding, appConfig)) {
17
+ return {
18
+ allowed: false,
19
+ reason: `Knowledge space "${space.meta.key}" is not bound in the resolved app config.`
20
+ };
21
+ }
22
+ if (context.operation === "write" && this.disallowedWrite.has(space.meta.category)) {
23
+ return {
24
+ allowed: false,
25
+ reason: `Knowledge space "${space.meta.key}" is category "${space.meta.category}" and is read-only.`
26
+ };
27
+ }
28
+ if (this.requireWorkflowBinding && context.workflowName) {
29
+ const allowedWorkflows = binding.scope?.workflows;
30
+ if (allowedWorkflows && !allowedWorkflows.includes(context.workflowName)) {
31
+ return {
32
+ allowed: false,
33
+ reason: `Workflow "${context.workflowName}" is not authorized to access knowledge space "${space.meta.key}".`
34
+ };
35
+ }
36
+ }
37
+ if (this.requireAgentBinding && context.agentName) {
38
+ const allowedAgents = binding.scope?.agents;
39
+ if (allowedAgents && !allowedAgents.includes(context.agentName)) {
40
+ return {
41
+ allowed: false,
42
+ reason: `Agent "${context.agentName}" is not authorized to access knowledge space "${space.meta.key}".`
43
+ };
44
+ }
45
+ }
46
+ if (space.meta.category === "ephemeral") {
47
+ return {
48
+ allowed: true,
49
+ severity: "warning",
50
+ reason: `Knowledge space "${space.meta.key}" is ephemeral; results may be transient.`
51
+ };
52
+ }
53
+ return { allowed: true };
54
+ }
55
+ isSpaceBound(resolved, appConfig) {
56
+ return appConfig.knowledge.some((entry) => entry.space.meta.key === resolved.space.meta.key && (resolved.space.meta.version == null || entry.space.meta.version === resolved.space.meta.version));
57
+ }
58
+ }
59
+ // src/retriever/static-retriever.ts
60
+ class StaticRetriever {
61
+ content;
62
+ constructor(config) {
63
+ this.content = config.content instanceof Map ? config.content : new Map(Object.entries(config.content));
64
+ }
65
+ async retrieve(query, options) {
66
+ const content = this.content.get(options.spaceKey);
67
+ if (!content)
68
+ return [];
69
+ const queryLower = query.toLowerCase();
70
+ const lines = content.split(`
71
+ `).filter((line) => line.trim());
72
+ const results = [];
73
+ for (const line of lines) {
74
+ if (line.toLowerCase().includes(queryLower)) {
75
+ results.push({
76
+ content: line,
77
+ source: options.spaceKey,
78
+ score: 1,
79
+ metadata: { type: "static" }
80
+ });
81
+ }
82
+ }
83
+ return results.slice(0, options.topK ?? 5);
84
+ }
85
+ async getStatic(spaceKey) {
86
+ return this.content.get(spaceKey) ?? null;
87
+ }
88
+ supportsSpace(spaceKey) {
89
+ return this.content.has(spaceKey);
90
+ }
91
+ listSpaces() {
92
+ return [...this.content.keys()];
93
+ }
94
+ }
95
+ function createStaticRetriever(content) {
96
+ return new StaticRetriever({ content });
97
+ }
98
+
99
+ // src/retriever/vector-retriever.ts
100
+ class VectorRetriever {
101
+ config;
102
+ spaceCollections;
103
+ staticContent;
104
+ constructor(config) {
105
+ this.config = config;
106
+ this.spaceCollections = config.spaceCollections instanceof Map ? config.spaceCollections : new Map(Object.entries(config.spaceCollections));
107
+ this.staticContent = config.staticContent ? config.staticContent instanceof Map ? config.staticContent : new Map(Object.entries(config.staticContent)) : new Map;
108
+ }
109
+ async retrieve(query, options) {
110
+ const collection = this.spaceCollections.get(options.spaceKey);
111
+ if (!collection) {
112
+ return [];
113
+ }
114
+ const embedding = await this.config.embeddings.embedQuery(query);
115
+ const results = await this.config.vectorStore.search({
116
+ collection,
117
+ vector: embedding.vector,
118
+ topK: options.topK ?? this.config.defaultTopK ?? 5,
119
+ namespace: options.tenantId,
120
+ filter: options.filter
121
+ });
122
+ const minScore = options.minScore ?? this.config.defaultMinScore ?? 0;
123
+ const filtered = results.filter((r) => r.score >= minScore);
124
+ return filtered.map((result) => ({
125
+ content: this.extractContent(result.payload),
126
+ source: result.id,
127
+ score: result.score,
128
+ metadata: result.payload
129
+ }));
130
+ }
131
+ async getStatic(spaceKey) {
132
+ return this.staticContent.get(spaceKey) ?? null;
133
+ }
134
+ supportsSpace(spaceKey) {
135
+ return this.spaceCollections.has(spaceKey);
136
+ }
137
+ listSpaces() {
138
+ return [...this.spaceCollections.keys()];
139
+ }
140
+ extractContent(payload) {
141
+ if (!payload)
142
+ return "";
143
+ if (typeof payload.text === "string")
144
+ return payload.text;
145
+ if (typeof payload.content === "string")
146
+ return payload.content;
147
+ return JSON.stringify(payload);
148
+ }
149
+ }
150
+ function createVectorRetriever(config) {
151
+ return new VectorRetriever(config);
152
+ }
153
+ // src/query/service.ts
154
+ class KnowledgeQueryService {
155
+ embeddings;
156
+ vectorStore;
157
+ llm;
158
+ config;
159
+ constructor(embeddings, vectorStore, llm, config) {
160
+ this.embeddings = embeddings;
161
+ this.vectorStore = vectorStore;
162
+ this.llm = llm;
163
+ this.config = config;
164
+ }
165
+ async query(question) {
166
+ const embedding = await this.embeddings.embedQuery(question);
167
+ const results = await this.vectorStore.search({
168
+ collection: this.config.collection,
169
+ vector: embedding.vector,
170
+ topK: this.config.topK ?? 5,
171
+ namespace: this.config.namespace,
172
+ filter: undefined
173
+ });
174
+ const context = buildContext(results);
175
+ const messages = this.buildMessages(question, context);
176
+ const response = await this.llm.chat(messages);
177
+ return {
178
+ answer: response.message.content.map((part) => ("text" in part) ? part.text : "").join(""),
179
+ references: results.map((result) => ({
180
+ ...result,
181
+ text: extractText(result)
182
+ })),
183
+ usage: response.usage
184
+ };
185
+ }
186
+ buildMessages(question, context) {
187
+ const systemPrompt = this.config.systemPrompt ?? "You are a knowledge assistant that answers questions using the provided context. Cite relevant sources if possible.";
188
+ return [
189
+ {
190
+ role: "system",
191
+ content: [{ type: "text", text: systemPrompt }]
192
+ },
193
+ {
194
+ role: "user",
195
+ content: [
196
+ {
197
+ type: "text",
198
+ text: `Question:
199
+ ${question}
200
+
201
+ Context:
202
+ ${context}`
203
+ }
204
+ ]
205
+ }
206
+ ];
207
+ }
208
+ }
209
+ function buildContext(results) {
210
+ if (results.length === 0) {
211
+ return "No relevant documents found.";
212
+ }
213
+ return results.map((result, index) => {
214
+ const text = extractText(result);
215
+ return `Source ${index + 1} (score: ${result.score.toFixed(3)}):
216
+ ${text}`;
217
+ }).join(`
218
+
219
+ `);
220
+ }
221
+ function extractText(result) {
222
+ const payload = result.payload ?? {};
223
+ if (typeof payload.text === "string")
224
+ return payload.text;
225
+ if (typeof payload.content === "string")
226
+ return payload.content;
227
+ return JSON.stringify(payload);
228
+ }
229
+ // src/ingestion/document-processor.ts
230
+ import { Buffer as Buffer2 } from "buffer";
231
+
232
+ class DocumentProcessor {
233
+ extractors = new Map;
234
+ constructor() {
235
+ this.registerExtractor("text/plain", this.extractText.bind(this));
236
+ this.registerExtractor("application/json", this.extractJson.bind(this));
237
+ }
238
+ registerExtractor(mimeType, extractor) {
239
+ this.extractors.set(mimeType.toLowerCase(), extractor);
240
+ }
241
+ async process(document) {
242
+ const extractor = this.extractors.get(document.mimeType.toLowerCase()) ?? this.extractors.get("*/*");
243
+ if (!extractor) {
244
+ throw new Error(`No extractor registered for mime type ${document.mimeType}`);
245
+ }
246
+ const fragments = await extractor(document);
247
+ if (fragments.length === 0) {
248
+ return [
249
+ {
250
+ id: `${document.id}:0`,
251
+ documentId: document.id,
252
+ text: "",
253
+ metadata: document.metadata
254
+ }
255
+ ];
256
+ }
257
+ return fragments;
258
+ }
259
+ async extractText(document) {
260
+ const text = Buffer2.from(document.data).toString("utf-8");
261
+ return [
262
+ {
263
+ id: `${document.id}:0`,
264
+ documentId: document.id,
265
+ text,
266
+ metadata: document.metadata
267
+ }
268
+ ];
269
+ }
270
+ async extractJson(document) {
271
+ const text = Buffer2.from(document.data).toString("utf-8");
272
+ try {
273
+ const json = JSON.parse(text);
274
+ return [
275
+ {
276
+ id: `${document.id}:0`,
277
+ documentId: document.id,
278
+ text: JSON.stringify(json, null, 2),
279
+ metadata: {
280
+ ...document.metadata,
281
+ contentType: "application/json"
282
+ }
283
+ }
284
+ ];
285
+ } catch {
286
+ return this.extractText(document);
287
+ }
288
+ }
289
+ }
290
+
291
+ // src/ingestion/embedding-service.ts
292
+ class EmbeddingService {
293
+ provider;
294
+ batchSize;
295
+ constructor(provider, batchSize = 16) {
296
+ this.provider = provider;
297
+ this.batchSize = batchSize;
298
+ }
299
+ async embedFragments(fragments) {
300
+ const results = [];
301
+ for (let i = 0;i < fragments.length; i += this.batchSize) {
302
+ const slice = fragments.slice(i, i + this.batchSize);
303
+ const documents = slice.map((fragment) => ({
304
+ id: fragment.id,
305
+ text: fragment.text,
306
+ metadata: fragment.metadata
307
+ }));
308
+ const embeddings = await this.provider.embedDocuments(documents);
309
+ results.push(...embeddings);
310
+ }
311
+ return results;
312
+ }
313
+ }
314
+
315
+ // src/ingestion/vector-indexer.ts
316
+ class VectorIndexer {
317
+ provider;
318
+ config;
319
+ constructor(provider, config) {
320
+ this.provider = provider;
321
+ this.config = config;
322
+ }
323
+ async upsert(fragments, embeddings) {
324
+ const documents = embeddings.map((embedding) => {
325
+ const fragment = fragments.find((f) => f.id === embedding.id);
326
+ return {
327
+ id: embedding.id,
328
+ vector: embedding.vector,
329
+ payload: {
330
+ ...this.config.metadata,
331
+ ...fragment?.metadata ?? {},
332
+ documentId: fragment?.documentId
333
+ },
334
+ namespace: this.config.namespace
335
+ };
336
+ });
337
+ const request = {
338
+ collection: this.config.collection,
339
+ documents
340
+ };
341
+ await this.provider.upsert(request);
342
+ }
343
+ }
344
+
345
+ // src/ingestion/gmail-adapter.ts
346
+ class GmailIngestionAdapter {
347
+ gmail;
348
+ processor;
349
+ embeddings;
350
+ indexer;
351
+ constructor(gmail, processor, embeddings, indexer) {
352
+ this.gmail = gmail;
353
+ this.processor = processor;
354
+ this.embeddings = embeddings;
355
+ this.indexer = indexer;
356
+ }
357
+ async syncThreads(query) {
358
+ const threads = await this.gmail.listThreads(query);
359
+ for (const thread of threads) {
360
+ await this.ingestThread(thread);
361
+ }
362
+ }
363
+ async ingestThread(thread) {
364
+ const document = this.toRawDocument(thread);
365
+ const fragments = await this.processor.process(document);
366
+ const embeddings = await this.embeddings.embedFragments(fragments);
367
+ await this.indexer.upsert(fragments, embeddings);
368
+ }
369
+ toRawDocument(thread) {
370
+ const content = composeThreadText(thread);
371
+ return {
372
+ id: thread.id,
373
+ mimeType: "text/plain",
374
+ data: Buffer.from(content, "utf-8"),
375
+ metadata: {
376
+ subject: thread.subject ?? "",
377
+ participants: thread.participants.map((p) => p.email).join(", "),
378
+ updatedAt: thread.updatedAt.toISOString()
379
+ }
380
+ };
381
+ }
382
+ }
383
+ function composeThreadText(thread) {
384
+ const header = [
385
+ `Subject: ${thread.subject ?? ""}`,
386
+ `Snippet: ${thread.snippet ?? ""}`
387
+ ];
388
+ const messageTexts = thread.messages.map((message) => {
389
+ const parts = [
390
+ `From: ${formatAddress(message.from)}`,
391
+ `To: ${message.to.map(formatAddress).join(", ")}`
392
+ ];
393
+ if (message.sentAt) {
394
+ parts.push(`Date: ${message.sentAt.toISOString()}`);
395
+ }
396
+ const body = message.textBody ?? stripHtml(message.htmlBody ?? "");
397
+ return `${parts.join(`
398
+ `)}
399
+
400
+ ${body ?? ""}`;
401
+ });
402
+ return [...header, ...messageTexts].join(`
403
+
404
+ ---
405
+
406
+ `);
407
+ }
408
+ function formatAddress(address) {
409
+ return address.name ? `${address.name} <${address.email}>` : address.email;
410
+ }
411
+ function stripHtml(html) {
412
+ return html.replace(/<[^>]+>/g, " ");
413
+ }
414
+
415
+ // src/ingestion/storage-adapter.ts
416
+ class StorageIngestionAdapter {
417
+ processor;
418
+ embeddings;
419
+ indexer;
420
+ constructor(processor, embeddings, indexer) {
421
+ this.processor = processor;
422
+ this.embeddings = embeddings;
423
+ this.indexer = indexer;
424
+ }
425
+ async ingestObject(object) {
426
+ if (!("data" in object) || !object.data) {
427
+ throw new Error("Storage ingestion requires object data");
428
+ }
429
+ const raw = {
430
+ id: object.key,
431
+ mimeType: object.contentType ?? "application/octet-stream",
432
+ data: object.data,
433
+ metadata: {
434
+ bucket: object.bucket,
435
+ checksum: object.checksum ?? ""
436
+ }
437
+ };
438
+ const fragments = await this.processor.process(raw);
439
+ const embeddings = await this.embeddings.embedFragments(fragments);
440
+ await this.indexer.upsert(fragments, embeddings);
441
+ }
442
+ }
443
+ export {
444
+ createVectorRetriever,
445
+ createStaticRetriever,
446
+ VectorRetriever,
447
+ VectorIndexer,
448
+ StorageIngestionAdapter,
449
+ StaticRetriever,
450
+ KnowledgeQueryService,
451
+ KnowledgeAccessGuard,
452
+ GmailIngestionAdapter,
453
+ EmbeddingService,
454
+ DocumentProcessor
455
+ };
@@ -1,25 +1,23 @@
1
- //#region src/ingestion/document-processor.d.ts
2
- interface RawDocument {
3
- id: string;
4
- mimeType: string;
5
- data: Uint8Array;
6
- metadata?: Record<string, string>;
1
+ export interface RawDocument {
2
+ id: string;
3
+ mimeType: string;
4
+ data: Uint8Array;
5
+ metadata?: Record<string, string>;
7
6
  }
8
- interface DocumentFragment {
9
- id: string;
10
- documentId: string;
11
- text: string;
12
- metadata?: Record<string, string>;
7
+ export interface DocumentFragment {
8
+ id: string;
9
+ documentId: string;
10
+ text: string;
11
+ metadata?: Record<string, string>;
13
12
  }
14
13
  type Extractor = (input: RawDocument) => Promise<DocumentFragment[]>;
15
- declare class DocumentProcessor {
16
- private readonly extractors;
17
- constructor();
18
- registerExtractor(mimeType: string, extractor: Extractor): void;
19
- process(document: RawDocument): Promise<DocumentFragment[]>;
20
- private extractText;
21
- private extractJson;
14
+ export declare class DocumentProcessor {
15
+ private readonly extractors;
16
+ constructor();
17
+ registerExtractor(mimeType: string, extractor: Extractor): void;
18
+ process(document: RawDocument): Promise<DocumentFragment[]>;
19
+ private extractText;
20
+ private extractJson;
22
21
  }
23
- //#endregion
24
- export { DocumentFragment, DocumentProcessor, RawDocument };
22
+ export {};
25
23
  //# sourceMappingURL=document-processor.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"document-processor.d.ts","names":[],"sources":["../../src/ingestion/document-processor.ts"],"sourcesContent":[],"mappings":";UAEiB,WAAA;EAAA,EAAA,EAAA,MAAA;EAOA,QAAA,EAAA,MAAA;EAOZ,IAAA,EAXG,UAWM;EAAW,QAAA,CAAA,EAVZ,MAUY,CAAA,MAAA,EAAA,MAAA,CAAA;;AAAgB,UAPxB,gBAAA,CAOwB;EAAO,EAAA,EAAA,MAAA;EAEnC,UAAA,EAAA,MAAA;EAQoC,IAAA,EAAA,MAAA;EAIvB,QAAA,CAAA,EAjBb,MAiBa,CAAA,MAAA,EAAA,MAAA,CAAA;;KAdrB,SAAA,GAcmC,CAAA,KAAA,EAdf,WAce,EAAA,GAdC,OAcD,CAdS,gBAcT,EAAA,CAAA;AAAO,cAZlC,iBAAA,CAYkC;;;iDAJE;oBAIvB,cAAc,QAAQ"}
1
+ {"version":3,"file":"document-processor.d.ts","sourceRoot":"","sources":["../../src/ingestion/document-processor.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACnC;AAED,KAAK,SAAS,GAAG,CAAC,KAAK,EAAE,WAAW,KAAK,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;AAErE,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgC;;IAO3D,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,GAAG,IAAI;IAIzD,OAAO,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAuBnD,WAAW;YAcX,WAAW;CAqB1B"}
@@ -1,55 +1,65 @@
1
- import { Buffer } from "node:buffer";
1
+ // @bun
2
+ // src/ingestion/document-processor.ts
3
+ import { Buffer } from "buffer";
2
4
 
3
- //#region src/ingestion/document-processor.ts
4
- var DocumentProcessor = class {
5
- extractors = /* @__PURE__ */ new Map();
6
- constructor() {
7
- this.registerExtractor("text/plain", this.extractText.bind(this));
8
- this.registerExtractor("application/json", this.extractJson.bind(this));
9
- }
10
- registerExtractor(mimeType, extractor) {
11
- this.extractors.set(mimeType.toLowerCase(), extractor);
12
- }
13
- async process(document) {
14
- const extractor = this.extractors.get(document.mimeType.toLowerCase()) ?? this.extractors.get("*/*");
15
- if (!extractor) throw new Error(`No extractor registered for mime type ${document.mimeType}`);
16
- const fragments = await extractor(document);
17
- if (fragments.length === 0) return [{
18
- id: `${document.id}:0`,
19
- documentId: document.id,
20
- text: "",
21
- metadata: document.metadata
22
- }];
23
- return fragments;
24
- }
25
- async extractText(document) {
26
- const text = Buffer.from(document.data).toString("utf-8");
27
- return [{
28
- id: `${document.id}:0`,
29
- documentId: document.id,
30
- text,
31
- metadata: document.metadata
32
- }];
33
- }
34
- async extractJson(document) {
35
- const text = Buffer.from(document.data).toString("utf-8");
36
- try {
37
- const json = JSON.parse(text);
38
- return [{
39
- id: `${document.id}:0`,
40
- documentId: document.id,
41
- text: JSON.stringify(json, null, 2),
42
- metadata: {
43
- ...document.metadata,
44
- contentType: "application/json"
45
- }
46
- }];
47
- } catch {
48
- return this.extractText(document);
49
- }
50
- }
5
+ class DocumentProcessor {
6
+ extractors = new Map;
7
+ constructor() {
8
+ this.registerExtractor("text/plain", this.extractText.bind(this));
9
+ this.registerExtractor("application/json", this.extractJson.bind(this));
10
+ }
11
+ registerExtractor(mimeType, extractor) {
12
+ this.extractors.set(mimeType.toLowerCase(), extractor);
13
+ }
14
+ async process(document) {
15
+ const extractor = this.extractors.get(document.mimeType.toLowerCase()) ?? this.extractors.get("*/*");
16
+ if (!extractor) {
17
+ throw new Error(`No extractor registered for mime type ${document.mimeType}`);
18
+ }
19
+ const fragments = await extractor(document);
20
+ if (fragments.length === 0) {
21
+ return [
22
+ {
23
+ id: `${document.id}:0`,
24
+ documentId: document.id,
25
+ text: "",
26
+ metadata: document.metadata
27
+ }
28
+ ];
29
+ }
30
+ return fragments;
31
+ }
32
+ async extractText(document) {
33
+ const text = Buffer.from(document.data).toString("utf-8");
34
+ return [
35
+ {
36
+ id: `${document.id}:0`,
37
+ documentId: document.id,
38
+ text,
39
+ metadata: document.metadata
40
+ }
41
+ ];
42
+ }
43
+ async extractJson(document) {
44
+ const text = Buffer.from(document.data).toString("utf-8");
45
+ try {
46
+ const json = JSON.parse(text);
47
+ return [
48
+ {
49
+ id: `${document.id}:0`,
50
+ documentId: document.id,
51
+ text: JSON.stringify(json, null, 2),
52
+ metadata: {
53
+ ...document.metadata,
54
+ contentType: "application/json"
55
+ }
56
+ }
57
+ ];
58
+ } catch {
59
+ return this.extractText(document);
60
+ }
61
+ }
62
+ }
63
+ export {
64
+ DocumentProcessor
51
65
  };
52
-
53
- //#endregion
54
- export { DocumentProcessor };
55
- //# sourceMappingURL=document-processor.js.map
@@ -1,13 +1,9 @@
1
- import { DocumentFragment } from "./document-processor.js";
2
- import { EmbeddingProvider, EmbeddingResult } from "@contractspec/lib.contracts";
3
-
4
- //#region src/ingestion/embedding-service.d.ts
5
- declare class EmbeddingService {
6
- private readonly provider;
7
- private readonly batchSize;
8
- constructor(provider: EmbeddingProvider, batchSize?: number);
9
- embedFragments(fragments: DocumentFragment[]): Promise<EmbeddingResult[]>;
1
+ import type { EmbeddingProvider, EmbeddingResult } from '@contractspec/lib.contracts';
2
+ import type { DocumentFragment } from './document-processor';
3
+ export declare class EmbeddingService {
4
+ private readonly provider;
5
+ private readonly batchSize;
6
+ constructor(provider: EmbeddingProvider, batchSize?: number);
7
+ embedFragments(fragments: DocumentFragment[]): Promise<EmbeddingResult[]>;
10
8
  }
11
- //#endregion
12
- export { EmbeddingService };
13
9
  //# sourceMappingURL=embedding-service.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"embedding-service.d.ts","names":[],"sources":["../../src/ingestion/embedding-service.ts"],"sourcesContent":[],"mappings":";;;;cAOa,gBAAA;;EAAA,iBAAA,SAAgB;EAIL,WAAA,CAAA,QAAA,EAAA,iBAAA,EAAA,SAAA,CAAA,EAAA,MAAA;EAMT,cAAA,CAAA,SAAA,EAAA,gBAAA,EAAA,CAAA,EACV,OADU,CACF,eADE,EAAA,CAAA"}
1
+ {"version":3,"file":"embedding-service.d.ts","sourceRoot":"","sources":["../../src/ingestion/embedding-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,iBAAiB,EACjB,eAAe,EAChB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,QAAQ,EAAE,iBAAiB,EAAE,SAAS,SAAK;IAKjD,cAAc,CAClB,SAAS,EAAE,gBAAgB,EAAE,GAC5B,OAAO,CAAC,eAAe,EAAE,CAAC;CAc9B"}