@contractspec/lib.knowledge 1.57.0 → 1.59.0

package/dist/node/ingestion/embedding-service.js

@@ -0,0 +1,26 @@
+// src/ingestion/embedding-service.ts
+class EmbeddingService {
+  provider;
+  batchSize;
+  constructor(provider, batchSize = 16) {
+    this.provider = provider;
+    this.batchSize = batchSize;
+  }
+  async embedFragments(fragments) {
+    const results = [];
+    for (let i = 0;i < fragments.length; i += this.batchSize) {
+      const slice = fragments.slice(i, i + this.batchSize);
+      const documents = slice.map((fragment) => ({
+        id: fragment.id,
+        text: fragment.text,
+        metadata: fragment.metadata
+      }));
+      const embeddings = await this.provider.embedDocuments(documents);
+      results.push(...embeddings);
+    }
+    return results;
+  }
+}
+export {
+  EmbeddingService
+};

package/dist/node/ingestion/gmail-adapter.js

@@ -0,0 +1,72 @@
+// src/ingestion/gmail-adapter.ts
+class GmailIngestionAdapter {
+  gmail;
+  processor;
+  embeddings;
+  indexer;
+  constructor(gmail, processor, embeddings, indexer) {
+    this.gmail = gmail;
+    this.processor = processor;
+    this.embeddings = embeddings;
+    this.indexer = indexer;
+  }
+  async syncThreads(query) {
+    const threads = await this.gmail.listThreads(query);
+    for (const thread of threads) {
+      await this.ingestThread(thread);
+    }
+  }
+  async ingestThread(thread) {
+    const document = this.toRawDocument(thread);
+    const fragments = await this.processor.process(document);
+    const embeddings = await this.embeddings.embedFragments(fragments);
+    await this.indexer.upsert(fragments, embeddings);
+  }
+  toRawDocument(thread) {
+    const content = composeThreadText(thread);
+    return {
+      id: thread.id,
+      mimeType: "text/plain",
+      data: Buffer.from(content, "utf-8"),
+      metadata: {
+        subject: thread.subject ?? "",
+        participants: thread.participants.map((p) => p.email).join(", "),
+        updatedAt: thread.updatedAt.toISOString()
+      }
+    };
+  }
+}
+function composeThreadText(thread) {
+  const header = [
+    `Subject: ${thread.subject ?? ""}`,
+    `Snippet: ${thread.snippet ?? ""}`
+  ];
+  const messageTexts = thread.messages.map((message) => {
+    const parts = [
+      `From: ${formatAddress(message.from)}`,
+      `To: ${message.to.map(formatAddress).join(", ")}`
+    ];
+    if (message.sentAt) {
+      parts.push(`Date: ${message.sentAt.toISOString()}`);
+    }
+    const body = message.textBody ?? stripHtml(message.htmlBody ?? "");
+    return `${parts.join(`
+`)}
+
+${body ?? ""}`;
+  });
+  return [...header, ...messageTexts].join(`
+
+---
+
+`);
+}
+function formatAddress(address) {
+  return address.name ? `${address.name} <${address.email}>` : address.email;
+}
+function stripHtml(html) {
+  return html.replace(/<[^>]+>/g, " ");
+}
+export {
+  GmailIngestionAdapter
+};

package/dist/node/ingestion/index.js

@@ -0,0 +1,221 @@
+// src/ingestion/document-processor.ts
+import { Buffer as Buffer2 } from "node:buffer";
+
+class DocumentProcessor {
+  extractors = new Map;
+  constructor() {
+    this.registerExtractor("text/plain", this.extractText.bind(this));
+    this.registerExtractor("application/json", this.extractJson.bind(this));
+  }
+  registerExtractor(mimeType, extractor) {
+    this.extractors.set(mimeType.toLowerCase(), extractor);
+  }
+  async process(document) {
+    const extractor = this.extractors.get(document.mimeType.toLowerCase()) ?? this.extractors.get("*/*");
+    if (!extractor) {
+      throw new Error(`No extractor registered for mime type ${document.mimeType}`);
+    }
+    const fragments = await extractor(document);
+    if (fragments.length === 0) {
+      return [
+        {
+          id: `${document.id}:0`,
+          documentId: document.id,
+          text: "",
+          metadata: document.metadata
+        }
+      ];
+    }
+    return fragments;
+  }
+  async extractText(document) {
+    const text = Buffer2.from(document.data).toString("utf-8");
+    return [
+      {
+        id: `${document.id}:0`,
+        documentId: document.id,
+        text,
+        metadata: document.metadata
+      }
+    ];
+  }
+  async extractJson(document) {
+    const text = Buffer2.from(document.data).toString("utf-8");
+    try {
+      const json = JSON.parse(text);
+      return [
+        {
+          id: `${document.id}:0`,
+          documentId: document.id,
+          text: JSON.stringify(json, null, 2),
+          metadata: {
+            ...document.metadata,
+            contentType: "application/json"
+          }
+        }
+      ];
+    } catch {
+      return this.extractText(document);
+    }
+  }
+}
+
+// src/ingestion/embedding-service.ts
+class EmbeddingService {
+  provider;
+  batchSize;
+  constructor(provider, batchSize = 16) {
+    this.provider = provider;
+    this.batchSize = batchSize;
+  }
+  async embedFragments(fragments) {
+    const results = [];
+    for (let i = 0;i < fragments.length; i += this.batchSize) {
+      const slice = fragments.slice(i, i + this.batchSize);
+      const documents = slice.map((fragment) => ({
+        id: fragment.id,
+        text: fragment.text,
+        metadata: fragment.metadata
+      }));
+      const embeddings = await this.provider.embedDocuments(documents);
+      results.push(...embeddings);
+    }
+    return results;
+  }
+}
+
+// src/ingestion/vector-indexer.ts
+class VectorIndexer {
+  provider;
+  config;
+  constructor(provider, config) {
+    this.provider = provider;
+    this.config = config;
+  }
+  async upsert(fragments, embeddings) {
+    const documents = embeddings.map((embedding) => {
+      const fragment = fragments.find((f) => f.id === embedding.id);
+      return {
+        id: embedding.id,
+        vector: embedding.vector,
+        payload: {
+          ...this.config.metadata,
+          ...fragment?.metadata ?? {},
+          documentId: fragment?.documentId
+        },
+        namespace: this.config.namespace
+      };
+    });
+    const request = {
+      collection: this.config.collection,
+      documents
+    };
+    await this.provider.upsert(request);
+  }
+}
+
+// src/ingestion/gmail-adapter.ts
+class GmailIngestionAdapter {
+  gmail;
+  processor;
+  embeddings;
+  indexer;
+  constructor(gmail, processor, embeddings, indexer) {
+    this.gmail = gmail;
+    this.processor = processor;
+    this.embeddings = embeddings;
+    this.indexer = indexer;
+  }
+  async syncThreads(query) {
+    const threads = await this.gmail.listThreads(query);
+    for (const thread of threads) {
+      await this.ingestThread(thread);
+    }
+  }
+  async ingestThread(thread) {
+    const document = this.toRawDocument(thread);
+    const fragments = await this.processor.process(document);
+    const embeddings = await this.embeddings.embedFragments(fragments);
+    await this.indexer.upsert(fragments, embeddings);
+  }
+  toRawDocument(thread) {
+    const content = composeThreadText(thread);
+    return {
+      id: thread.id,
+      mimeType: "text/plain",
+      data: Buffer.from(content, "utf-8"),
+      metadata: {
+        subject: thread.subject ?? "",
+        participants: thread.participants.map((p) => p.email).join(", "),
+        updatedAt: thread.updatedAt.toISOString()
+      }
+    };
+  }
+}
+function composeThreadText(thread) {
+  const header = [
+    `Subject: ${thread.subject ?? ""}`,
+    `Snippet: ${thread.snippet ?? ""}`
+  ];
+  const messageTexts = thread.messages.map((message) => {
+    const parts = [
+      `From: ${formatAddress(message.from)}`,
+      `To: ${message.to.map(formatAddress).join(", ")}`
+    ];
+    if (message.sentAt) {
+      parts.push(`Date: ${message.sentAt.toISOString()}`);
+    }
+    const body = message.textBody ?? stripHtml(message.htmlBody ?? "");
+    return `${parts.join(`
+`)}
+
+${body ?? ""}`;
+  });
+  return [...header, ...messageTexts].join(`
+
+---
+
+`);
+}
+function formatAddress(address) {
+  return address.name ? `${address.name} <${address.email}>` : address.email;
+}
+function stripHtml(html) {
+  return html.replace(/<[^>]+>/g, " ");
+}
+
+// src/ingestion/storage-adapter.ts
+class StorageIngestionAdapter {
+  processor;
+  embeddings;
+  indexer;
+  constructor(processor, embeddings, indexer) {
+    this.processor = processor;
+    this.embeddings = embeddings;
+    this.indexer = indexer;
+  }
+  async ingestObject(object) {
+    if (!("data" in object) || !object.data) {
+      throw new Error("Storage ingestion requires object data");
+    }
+    const raw = {
+      id: object.key,
+      mimeType: object.contentType ?? "application/octet-stream",
+      data: object.data,
+      metadata: {
+        bucket: object.bucket,
+        checksum: object.checksum ?? ""
+      }
+    };
+    const fragments = await this.processor.process(raw);
+    const embeddings = await this.embeddings.embedFragments(fragments);
+    await this.indexer.upsert(fragments, embeddings);
+  }
+}
+export {
+  VectorIndexer,
+  StorageIngestionAdapter,
+  GmailIngestionAdapter,
+  EmbeddingService,
+  DocumentProcessor
+};

package/dist/node/ingestion/storage-adapter.js

@@ -0,0 +1,31 @@
+// src/ingestion/storage-adapter.ts
+class StorageIngestionAdapter {
+  processor;
+  embeddings;
+  indexer;
+  constructor(processor, embeddings, indexer) {
+    this.processor = processor;
+    this.embeddings = embeddings;
+    this.indexer = indexer;
+  }
+  async ingestObject(object) {
+    if (!("data" in object) || !object.data) {
+      throw new Error("Storage ingestion requires object data");
+    }
+    const raw = {
+      id: object.key,
+      mimeType: object.contentType ?? "application/octet-stream",
+      data: object.data,
+      metadata: {
+        bucket: object.bucket,
+        checksum: object.checksum ?? ""
+      }
+    };
+    const fragments = await this.processor.process(raw);
+    const embeddings = await this.embeddings.embedFragments(fragments);
+    await this.indexer.upsert(fragments, embeddings);
+  }
+}
+export {
+  StorageIngestionAdapter
+};

package/dist/node/ingestion/vector-indexer.js

@@ -0,0 +1,32 @@
+// src/ingestion/vector-indexer.ts
+class VectorIndexer {
+  provider;
+  config;
+  constructor(provider, config) {
+    this.provider = provider;
+    this.config = config;
+  }
+  async upsert(fragments, embeddings) {
+    const documents = embeddings.map((embedding) => {
+      const fragment = fragments.find((f) => f.id === embedding.id);
+      return {
+        id: embedding.id,
+        vector: embedding.vector,
+        payload: {
+          ...this.config.metadata,
+          ...fragment?.metadata ?? {},
+          documentId: fragment?.documentId
+        },
+        namespace: this.config.namespace
+      };
+    });
+    const request = {
+      collection: this.config.collection,
+      documents
+    };
+    await this.provider.upsert(request);
+  }
+}
+export {
+  VectorIndexer
+};

package/dist/node/query/index.js

@@ -0,0 +1,79 @@
+// src/query/service.ts
+class KnowledgeQueryService {
+  embeddings;
+  vectorStore;
+  llm;
+  config;
+  constructor(embeddings, vectorStore, llm, config) {
+    this.embeddings = embeddings;
+    this.vectorStore = vectorStore;
+    this.llm = llm;
+    this.config = config;
+  }
+  async query(question) {
+    const embedding = await this.embeddings.embedQuery(question);
+    const results = await this.vectorStore.search({
+      collection: this.config.collection,
+      vector: embedding.vector,
+      topK: this.config.topK ?? 5,
+      namespace: this.config.namespace,
+      filter: undefined
+    });
+    const context = buildContext(results);
+    const messages = this.buildMessages(question, context);
+    const response = await this.llm.chat(messages);
+    return {
+      answer: response.message.content.map((part) => ("text" in part) ? part.text : "").join(""),
+      references: results.map((result) => ({
+        ...result,
+        text: extractText(result)
+      })),
+      usage: response.usage
+    };
+  }
+  buildMessages(question, context) {
+    const systemPrompt = this.config.systemPrompt ?? "You are a knowledge assistant that answers questions using the provided context. Cite relevant sources if possible.";
+    return [
+      {
+        role: "system",
+        content: [{ type: "text", text: systemPrompt }]
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: `Question:
+${question}
+
+Context:
+${context}`
+          }
+        ]
+      }
+    ];
+  }
+}
+function buildContext(results) {
+  if (results.length === 0) {
+    return "No relevant documents found.";
+  }
+  return results.map((result, index) => {
+    const text = extractText(result);
+    return `Source ${index + 1} (score: ${result.score.toFixed(3)}):
+${text}`;
+  }).join(`
+
+`);
+}
+function extractText(result) {
+  const payload = result.payload ?? {};
+  if (typeof payload.text === "string")
+    return payload.text;
+  if (typeof payload.content === "string")
+    return payload.content;
+  return JSON.stringify(payload);
+}
+export {
+  KnowledgeQueryService
+};

package/dist/node/query/service.js

@@ -0,0 +1,79 @@
+// src/query/service.ts
+class KnowledgeQueryService {
+  embeddings;
+  vectorStore;
+  llm;
+  config;
+  constructor(embeddings, vectorStore, llm, config) {
+    this.embeddings = embeddings;
+    this.vectorStore = vectorStore;
+    this.llm = llm;
+    this.config = config;
+  }
+  async query(question) {
+    const embedding = await this.embeddings.embedQuery(question);
+    const results = await this.vectorStore.search({
+      collection: this.config.collection,
+      vector: embedding.vector,
+      topK: this.config.topK ?? 5,
+      namespace: this.config.namespace,
+      filter: undefined
+    });
+    const context = buildContext(results);
+    const messages = this.buildMessages(question, context);
+    const response = await this.llm.chat(messages);
+    return {
+      answer: response.message.content.map((part) => ("text" in part) ? part.text : "").join(""),
+      references: results.map((result) => ({
+        ...result,
+        text: extractText(result)
+      })),
+      usage: response.usage
+    };
+  }
+  buildMessages(question, context) {
+    const systemPrompt = this.config.systemPrompt ?? "You are a knowledge assistant that answers questions using the provided context. Cite relevant sources if possible.";
+    return [
+      {
+        role: "system",
+        content: [{ type: "text", text: systemPrompt }]
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: `Question:
+${question}
+
+Context:
+${context}`
+          }
+        ]
+      }
+    ];
+  }
+}
+function buildContext(results) {
+  if (results.length === 0) {
+    return "No relevant documents found.";
+  }
+  return results.map((result, index) => {
+    const text = extractText(result);
+    return `Source ${index + 1} (score: ${result.score.toFixed(3)}):
+${text}`;
+  }).join(`
+
+`);
+}
+function extractText(result) {
+  const payload = result.payload ?? {};
+  if (typeof payload.text === "string")
+    return payload.text;
+  if (typeof payload.content === "string")
+    return payload.content;
+  return JSON.stringify(payload);
+}
+export {
+  KnowledgeQueryService
+};

package/dist/node/retriever/index.js

@@ -0,0 +1,100 @@
+// src/retriever/static-retriever.ts
+class StaticRetriever {
+  content;
+  constructor(config) {
+    this.content = config.content instanceof Map ? config.content : new Map(Object.entries(config.content));
+  }
+  async retrieve(query, options) {
+    const content = this.content.get(options.spaceKey);
+    if (!content)
+      return [];
+    const queryLower = query.toLowerCase();
+    const lines = content.split(`
+`).filter((line) => line.trim());
+    const results = [];
+    for (const line of lines) {
+      if (line.toLowerCase().includes(queryLower)) {
+        results.push({
+          content: line,
+          source: options.spaceKey,
+          score: 1,
+          metadata: { type: "static" }
+        });
+      }
+    }
+    return results.slice(0, options.topK ?? 5);
+  }
+  async getStatic(spaceKey) {
+    return this.content.get(spaceKey) ?? null;
+  }
+  supportsSpace(spaceKey) {
+    return this.content.has(spaceKey);
+  }
+  listSpaces() {
+    return [...this.content.keys()];
+  }
+}
+function createStaticRetriever(content) {
+  return new StaticRetriever({ content });
+}
+
+// src/retriever/vector-retriever.ts
+class VectorRetriever {
+  config;
+  spaceCollections;
+  staticContent;
+  constructor(config) {
+    this.config = config;
+    this.spaceCollections = config.spaceCollections instanceof Map ? config.spaceCollections : new Map(Object.entries(config.spaceCollections));
+    this.staticContent = config.staticContent ? config.staticContent instanceof Map ? config.staticContent : new Map(Object.entries(config.staticContent)) : new Map;
+  }
+  async retrieve(query, options) {
+    const collection = this.spaceCollections.get(options.spaceKey);
+    if (!collection) {
+      return [];
+    }
+    const embedding = await this.config.embeddings.embedQuery(query);
+    const results = await this.config.vectorStore.search({
+      collection,
+      vector: embedding.vector,
+      topK: options.topK ?? this.config.defaultTopK ?? 5,
+      namespace: options.tenantId,
+      filter: options.filter
+    });
+    const minScore = options.minScore ?? this.config.defaultMinScore ?? 0;
+    const filtered = results.filter((r) => r.score >= minScore);
+    return filtered.map((result) => ({
+      content: this.extractContent(result.payload),
+      source: result.id,
+      score: result.score,
+      metadata: result.payload
+    }));
+  }
+  async getStatic(spaceKey) {
+    return this.staticContent.get(spaceKey) ?? null;
+  }
+  supportsSpace(spaceKey) {
+    return this.spaceCollections.has(spaceKey);
+  }
+  listSpaces() {
+    return [...this.spaceCollections.keys()];
+  }
+  extractContent(payload) {
+    if (!payload)
+      return "";
+    if (typeof payload.text === "string")
+      return payload.text;
+    if (typeof payload.content === "string")
+      return payload.content;
+    return JSON.stringify(payload);
+  }
+}
+function createVectorRetriever(config) {
+  return new VectorRetriever(config);
+}
+export {
+  createVectorRetriever,
+  createStaticRetriever,
+  VectorRetriever,
+  StaticRetriever
+};

package/dist/node/retriever/static-retriever.js

@@ -0,0 +1,43 @@
+// src/retriever/static-retriever.ts
+class StaticRetriever {
+  content;
+  constructor(config) {
+    this.content = config.content instanceof Map ? config.content : new Map(Object.entries(config.content));
+  }
+  async retrieve(query, options) {
+    const content = this.content.get(options.spaceKey);
+    if (!content)
+      return [];
+    const queryLower = query.toLowerCase();
+    const lines = content.split(`
+`).filter((line) => line.trim());
+    const results = [];
+    for (const line of lines) {
+      if (line.toLowerCase().includes(queryLower)) {
+        results.push({
+          content: line,
+          source: options.spaceKey,
+          score: 1,
+          metadata: { type: "static" }
+        });
+      }
+    }
+    return results.slice(0, options.topK ?? 5);
+  }
+  async getStatic(spaceKey) {
+    return this.content.get(spaceKey) ?? null;
+  }
+  supportsSpace(spaceKey) {
+    return this.content.has(spaceKey);
+  }
+  listSpaces() {
+    return [...this.content.keys()];
+  }
+}
+function createStaticRetriever(content) {
+  return new StaticRetriever({ content });
+}
+export {
+  createStaticRetriever,
+  StaticRetriever
+};