@workglow/dataset 0.0.109 → 0.0.113

package/dist/bun.js

@@ -1,228 +1,4 @@
 // @bun
-// src/util/DatasetSchema.ts
-function TypeTabularStorage(options = {}) {
-  return {
-    title: "Tabular Storage",
-    description: "Storage ID or instance for tabular data storage",
-    ...options,
-    format: "storage:tabular",
-    oneOf: [
-      { type: "string", title: "Storage ID" },
-      { title: "Storage Instance", additionalProperties: true }
-    ]
-  };
-}
-function TypeDocumentChunkDataset(options = {}) {
-  return {
-    title: "Document Chunk Dataset",
-    description: "Dataset ID or instance for document chunk data storage",
-    ...options,
-    format: "dataset:document-chunk",
-    anyOf: [
-      { type: "string", title: "Dataset ID" },
-      { title: "Dataset Instance", additionalProperties: true }
-    ]
-  };
-}
-function TypeDocumentDataset(options = {}) {
-  return {
-    title: "Document Dataset",
-    description: "Dataset ID or instance for document data storage",
-    ...options,
-    format: "dataset:document",
-    anyOf: [
-      { type: "string", title: "Dataset ID" },
-      { title: "Dataset Instance", additionalProperties: true }
-    ]
-  };
-}
-// src/document/Document.ts
-class Document {
-  doc_id;
-  metadata;
-  root;
-  chunks;
-  constructor(root, metadata, chunks = [], doc_id) {
-    this.doc_id = doc_id;
-    this.root = root;
-    this.metadata = metadata;
-    this.chunks = chunks || [];
-  }
-  setChunks(chunks) {
-    this.chunks = chunks;
-  }
-  getChunks() {
-    return this.chunks;
-  }
-  setDocId(doc_id) {
-    this.doc_id = doc_id;
-  }
-  findChunksByNodeId(nodeId) {
-    return this.chunks.filter((chunk) => chunk.nodePath.includes(nodeId));
-  }
-  toJSON() {
-    return {
-      metadata: this.metadata,
-      root: this.root,
-      chunks: this.chunks
-    };
-  }
-  static fromJSON(json, doc_id) {
-    const obj = JSON.parse(json);
-    return new Document(obj.root, obj.metadata, obj.chunks, doc_id);
-  }
-}
-// src/document/DocumentDataset.ts
-class DocumentDataset {
-  tabularStorage;
-  vectorStorage;
-  constructor(tabularStorage, vectorStorage) {
-    this.tabularStorage = tabularStorage;
-    this.vectorStorage = vectorStorage;
-  }
-  async upsert(document) {
-    const serialized = JSON.stringify(document.toJSON());
-    const insertEntity = {
-      doc_id: document.doc_id,
-      data: serialized
-    };
-    const entity = await this.tabularStorage.put(insertEntity);
-    if (document.doc_id !== entity.doc_id) {
-      document.setDocId(entity.doc_id);
-    }
-    return document;
-  }
-  async get(doc_id) {
-    const entity = await this.tabularStorage.get({ doc_id });
-    if (!entity) {
-      return;
-    }
-    return Document.fromJSON(entity.data, entity.doc_id);
-  }
-  async delete(doc_id) {
-    await this.tabularStorage.delete({ doc_id });
-  }
-  async getNode(doc_id, nodeId) {
-    const doc = await this.get(doc_id);
-    if (!doc) {
-      return;
-    }
-    const traverse = (node) => {
-      if (node.nodeId === nodeId) {
-        return node;
-      }
-      if (node.children && Array.isArray(node.children)) {
-        for (const child of node.children) {
-          const found = traverse(child);
-          if (found)
-            return found;
-        }
-      }
-      return;
-    };
-    return traverse(doc.root);
-  }
-  async getAncestors(doc_id, nodeId) {
-    const doc = await this.get(doc_id);
-    if (!doc) {
-      return [];
-    }
-    const path = [];
-    const findPath = (node) => {
-      path.push(node.nodeId);
-      if (node.nodeId === nodeId) {
-        return true;
-      }
-      if (node.children && Array.isArray(node.children)) {
-        for (const child of node.children) {
-          if (findPath(child)) {
-            return true;
-          }
-        }
-      }
-      path.pop();
-      return false;
-    };
-    if (!findPath(doc.root)) {
-      return [];
-    }
-    const ancestors = [];
-    let currentNode = doc.root;
-    ancestors.push(currentNode);
-    for (let i = 1;i < path.length; i++) {
-      const targetId = path[i];
-      if (currentNode.children && Array.isArray(currentNode.children)) {
-        const found = currentNode.children.find((child) => child.nodeId === targetId);
-        if (found) {
-          currentNode = found;
-          ancestors.push(currentNode);
-        } else {
-          break;
-        }
-      } else {
-        break;
-      }
-    }
-    return ancestors;
-  }
-  async getChunks(doc_id) {
-    const doc = await this.get(doc_id);
-    if (!doc) {
-      return [];
-    }
-    return doc.getChunks();
-  }
-  async findChunksByNodeId(doc_id, nodeId) {
-    const doc = await this.get(doc_id);
-    if (!doc) {
-      return [];
-    }
-    if (doc.findChunksByNodeId) {
-      return doc.findChunksByNodeId(nodeId);
-    }
-    const chunks = doc.getChunks();
-    return chunks.filter((chunk) => chunk.nodePath && chunk.nodePath.includes(nodeId));
-  }
-  async list() {
-    const entities = await this.tabularStorage.getAll();
-    if (!entities) {
-      return [];
-    }
-    return entities.map((e) => e.doc_id);
-  }
-  async search(query, options) {
-    return this.vectorStorage?.similaritySearch(query, options) || [];
-  }
-}
-// src/document/DocumentDatasetRegistry.ts
-import {
-  createServiceToken,
-  globalServiceRegistry,
-  registerInputResolver
-} from "@workglow/util";
-var DOCUMENT_DATASETS = createServiceToken("dataset.documents");
-if (!globalServiceRegistry.has(DOCUMENT_DATASETS)) {
-  globalServiceRegistry.register(DOCUMENT_DATASETS, () => new Map, true);
-}
-function getGlobalDocumentDatasets() {
-  return globalServiceRegistry.get(DOCUMENT_DATASETS);
-}
-function registerDocumentDataset(id, dataset) {
-  const datasets = getGlobalDocumentDatasets();
-  datasets.set(id, dataset);
-}
-function getDocumentDataset(id) {
-  return getGlobalDocumentDatasets().get(id);
-}
-async function resolveDocumentDatasetFromRegistry(id, format, registry) {
-  const datasets = registry.has(DOCUMENT_DATASETS) ? registry.get(DOCUMENT_DATASETS) : getGlobalDocumentDatasets();
-  const dataset = datasets.get(id);
-  if (!dataset) {
-    throw new Error(`Document dataset "${id}" not found in registry`);
-  }
-  return dataset;
-}
-registerInputResolver("dataset:document", resolveDocumentDatasetFromRegistry);
 // src/document/DocumentSchema.ts
 var NodeKind = {
   DOCUMENT: "document",
@@ -470,24 +246,31 @@ var TokenBudgetSchema = {
   required: ["maxTokensPerChunk", "overlapTokens", "reservedTokens"],
   additionalProperties: false
 };
-var ChunkEnrichmentSchema = {
+var DocumentMetadataSchema = {
   type: "object",
   properties: {
-    summary: {
+    title: {
       type: "string",
-      title: "Summary",
-      description: "Summary of the chunk content"
+      title: "Title",
+      description: "Document title"
     },
-    entities: {
-      type: "array",
-      items: EntitySchema,
-      title: "Entities",
-      description: "Named entities extracted from the chunk"
+    sourceUri: {
+      type: "string",
+      title: "Source URI",
+      description: "Original source URI of the document"
+    },
+    createdAt: {
+      type: "string",
+      title: "Created At",
+      description: "ISO timestamp of creation"
     }
   },
-  additionalProperties: false
+  required: ["title"],
+  additionalProperties: true
 };
-var ChunkNodeSchema = () => ({
+
+// src/chunk/ChunkSchema.ts
+var ChunkRecordSchema = () => ({
   type: "object",
   properties: {
     chunkId: {
@@ -516,45 +299,11 @@ var ChunkNodeSchema = () => ({
       title: "Depth",
       description: "Depth in the document tree"
     },
-    enrichment: ChunkEnrichmentSchema
-  },
-  required: ["chunkId", "doc_id", "text", "nodePath", "depth"],
-  additionalProperties: false
-});
-var ChunkMetadataSchema = {
-  type: "object",
-  properties: {
-    doc_id: {
-      type: "string",
-      title: "Document ID",
-      description: "ID of the parent document"
-    },
-    chunkId: {
-      type: "string",
-      title: "Chunk ID",
-      description: "Unique identifier for this chunk"
-    },
     leafNodeId: {
       type: "string",
       title: "Leaf Node ID",
       description: "ID of the leaf node this chunk belongs to"
     },
-    depth: {
-      type: "integer",
-      title: "Depth",
-      description: "Depth in the document tree"
-    },
-    text: {
-      type: "string",
-      title: "Text",
-      description: "Text content of the chunk"
-    },
-    nodePath: {
-      type: "array",
-      items: { type: "string" },
-      title: "Node Path",
-      description: "Node IDs from root to leaf"
-    },
     summary: {
       type: "string",
       title: "Summary",
@@ -565,107 +314,372 @@ var ChunkMetadataSchema = {
       items: EntitySchema,
       title: "Entities",
       description: "Named entities extracted from the chunk"
+    },
+    parentSummaries: {
+      type: "array",
+      items: { type: "string" },
+      title: "Parent Summaries",
+      description: "Summaries from ancestor nodes"
+    },
+    sectionTitles: {
+      type: "array",
+      items: { type: "string" },
+      title: "Section Titles",
+      description: "Titles of ancestor section nodes"
+    },
+    doc_title: {
+      type: "string",
+      title: "Document Title",
+      description: "Title of the parent document"
     }
   },
-  required: ["doc_id", "chunkId", "leafNodeId", "depth", "text", "nodePath"],
+  required: ["chunkId", "doc_id", "text", "nodePath", "depth"],
   additionalProperties: true
-};
-var ChunkMetadataArraySchema = {
+});
+var ChunkRecordArraySchema = {
   type: "array",
-  items: ChunkMetadataSchema,
-  title: "Chunk Metadata",
-  description: "Metadata for each chunk"
+  items: ChunkRecordSchema(),
+  title: "Chunk Records",
+  description: "Array of chunk records"
 };
-var EnrichedChunkMetadataSchema = {
+// src/chunk/ChunkVectorStorageSchema.ts
+import { TypedArraySchema } from "@workglow/util";
+var ChunkVectorStorageSchema = {
   type: "object",
   properties: {
-    doc_id: {
-      type: "string",
-      title: "Document ID",
-      description: "ID of the parent document"
-    },
-    chunkId: {
-      type: "string",
-      title: "Chunk ID",
-      description: "Unique identifier for this chunk"
-    },
-    leafNodeId: {
-      type: "string",
-      title: "Leaf Node ID",
-      description: "ID of the leaf node this chunk belongs to"
-    },
-    depth: {
-      type: "integer",
-      title: "Depth",
-      description: "Depth in the document tree"
-    },
-    text: {
-      type: "string",
-      title: "Text",
-      description: "Text content of the chunk"
-    },
-    nodePath: {
-      type: "array",
-      items: { type: "string" },
-      title: "Node Path",
-      description: "Node IDs from root to leaf"
-    },
-    summary: {
-      type: "string",
-      title: "Summary",
-      description: "Summary of the chunk content"
-    },
-    entities: {
-      type: "array",
-      items: EntitySchema,
-      title: "Entities",
-      description: "Named entities (rolled up from hierarchy)"
-    },
-    parentSummaries: {
-      type: "array",
-      items: { type: "string" },
-      title: "Parent Summaries",
-      description: "Summaries from ancestor nodes"
-    },
-    sectionTitles: {
-      type: "array",
-      items: { type: "string" },
-      title: "Section Titles",
-      description: "Titles of ancestor section nodes"
-    }
+    chunk_id: { type: "string", "x-auto-generated": true },
+    doc_id: { type: "string" },
+    vector: TypedArraySchema(),
+    metadata: { type: "object", format: "metadata", additionalProperties: true }
   },
-  required: ["doc_id", "chunkId", "leafNodeId", "depth", "text", "nodePath"],
-  additionalProperties: true
-};
-var EnrichedChunkMetadataArraySchema = {
-  type: "array",
-  items: EnrichedChunkMetadataSchema,
-  title: "Enriched Metadata",
-  description: "Metadata enriched with hierarchy information"
+  required: ["chunk_id", "doc_id", "vector", "metadata"],
+  additionalProperties: false
 };
-var DocumentMetadataSchema = {
+var ChunkVectorPrimaryKey = ["chunk_id"];
+// src/document/Document.ts
+class Document {
+  doc_id;
+  metadata;
+  root;
+  chunks;
+  constructor(root, metadata, chunks = [], doc_id) {
+    this.doc_id = doc_id;
+    this.root = root;
+    this.metadata = metadata;
+    this.chunks = chunks || [];
+  }
+  setChunks(chunks) {
+    this.chunks = chunks;
+  }
+  getChunks() {
+    return this.chunks;
+  }
+  setDocId(doc_id) {
+    this.doc_id = doc_id;
+  }
+  findChunksByNodeId(nodeId) {
+    return this.chunks.filter((chunk) => chunk.nodePath.includes(nodeId));
+  }
+  toJSON() {
+    return {
+      metadata: this.metadata,
+      root: this.root,
+      chunks: this.chunks
+    };
+  }
+  static fromJSON(json, doc_id) {
+    const obj = JSON.parse(json);
+    return new Document(obj.root, obj.metadata, obj.chunks, doc_id);
+  }
+}
+
+// src/knowledge-base/KnowledgeBase.ts
+class KnowledgeBase {
+  name;
+  tabularStorage;
+  chunkStorage;
+  constructor(name, documentStorage, chunkStorage) {
+    this.name = name;
+    this.tabularStorage = documentStorage;
+    this.chunkStorage = chunkStorage;
+  }
+  async upsertDocument(document) {
+    const serialized = JSON.stringify(document.toJSON());
+    const insertEntity = {
+      doc_id: document.doc_id,
+      data: serialized
+    };
+    const entity = await this.tabularStorage.put(insertEntity);
+    if (document.doc_id !== entity.doc_id) {
+      document.setDocId(entity.doc_id);
+    }
+    return document;
+  }
+  async getDocument(doc_id) {
+    const entity = await this.tabularStorage.get({ doc_id });
+    if (!entity) {
+      return;
+    }
+    return Document.fromJSON(entity.data, entity.doc_id);
+  }
+  async deleteDocument(doc_id) {
+    await this.deleteChunksForDocument(doc_id);
+    await this.tabularStorage.delete({ doc_id });
+  }
+  async listDocuments() {
+    const entities = await this.tabularStorage.getAll();
+    if (!entities) {
+      return [];
+    }
+    return entities.map((e) => e.doc_id);
+  }
+  async getNode(doc_id, nodeId) {
+    const doc = await this.getDocument(doc_id);
+    if (!doc) {
+      return;
+    }
+    const traverse = (node) => {
+      if (node.nodeId === nodeId) {
+        return node;
+      }
+      if ("children" in node && Array.isArray(node.children)) {
+        for (const child of node.children) {
+          const found = traverse(child);
+          if (found)
+            return found;
+        }
+      }
+      return;
+    };
+    return traverse(doc.root);
+  }
+  async getAncestors(doc_id, nodeId) {
+    const doc = await this.getDocument(doc_id);
+    if (!doc) {
+      return [];
+    }
+    const path = [];
+    const findPath = (node) => {
+      path.push(node.nodeId);
+      if (node.nodeId === nodeId) {
+        return true;
+      }
+      if ("children" in node && Array.isArray(node.children)) {
+        for (const child of node.children) {
+          if (findPath(child)) {
+            return true;
+          }
+        }
+      }
+      path.pop();
+      return false;
+    };
+    if (!findPath(doc.root)) {
+      return [];
+    }
+    const ancestors = [];
+    let currentNode = doc.root;
+    ancestors.push(currentNode);
+    for (let i = 1;i < path.length; i++) {
+      const targetId = path[i];
+      if ("children" in currentNode && Array.isArray(currentNode.children)) {
+        const found = currentNode.children.find((child) => child.nodeId === targetId);
+        if (found) {
+          currentNode = found;
+          ancestors.push(currentNode);
+        } else {
+          break;
+        }
+      } else {
+        break;
+      }
+    }
+    return ancestors;
+  }
+  async upsertChunk(chunk) {
+    if (chunk.vector.length !== this.getVectorDimensions()) {
+      throw new Error(`Vector dimension mismatch: expected ${this.getVectorDimensions()}, got ${chunk.vector.length}.`);
+    }
+    return this.chunkStorage.put(chunk);
+  }
+  async upsertChunksBulk(chunks) {
+    const expected = this.getVectorDimensions();
+    for (const chunk of chunks) {
+      if (chunk.vector.length !== expected) {
+        throw new Error(`Vector dimension mismatch: expected ${expected}, got ${chunk.vector.length}.`);
+      }
+    }
+    return this.chunkStorage.putBulk(chunks);
+  }
+  async deleteChunksForDocument(doc_id) {
+    await this.chunkStorage.deleteSearch({ doc_id });
+  }
+  async getChunksForDocument(doc_id) {
+    const results = await this.chunkStorage.query({ doc_id });
+    return results ?? [];
+  }
+  async similaritySearch(query, options) {
+    return this.chunkStorage.similaritySearch(query, options);
+  }
+  async hybridSearch(query, options) {
+    if (typeof this.chunkStorage.hybridSearch !== "function") {
+      throw new Error("Hybrid search is not supported by the configured chunk storage backend. " + "Please use a vector storage implementation that provides `hybridSearch`.");
+    }
+    return this.chunkStorage.hybridSearch(query, options);
+  }
+  async prepareReindex(doc_id) {
+    const doc = await this.getDocument(doc_id);
+    if (!doc) {
+      return;
+    }
+    await this.deleteChunksForDocument(doc_id);
+    return doc;
+  }
+  async setupDatabase() {
+    await this.tabularStorage.setupDatabase();
+    await this.chunkStorage.setupDatabase();
+  }
+  destroy() {
+    this.tabularStorage.destroy();
+    this.chunkStorage.destroy();
+  }
+  async getChunk(chunk_id) {
+    return this.chunkStorage.get({ chunk_id });
+  }
+  async put(chunk) {
+    return this.chunkStorage.put(chunk);
+  }
+  async putBulk(chunks) {
+    return this.chunkStorage.putBulk(chunks);
+  }
+  async getAllChunks() {
+    return this.chunkStorage.getAll();
+  }
+  async chunkCount() {
+    return this.chunkStorage.size();
+  }
+  async clearChunks() {
+    return this.chunkStorage.deleteAll();
+  }
+  getVectorDimensions() {
+    return this.chunkStorage.getVectorDimensions();
+  }
+  async getDocumentChunks(doc_id) {
+    const doc = await this.getDocument(doc_id);
+    if (!doc) {
+      return [];
+    }
+    return doc.getChunks();
+  }
+  async findChunksByNodeId(doc_id, nodeId) {
+    const doc = await this.getDocument(doc_id);
+    if (!doc) {
+      return [];
+    }
+    return doc.findChunksByNodeId(nodeId);
+  }
+}
+// src/knowledge-base/KnowledgeBaseRegistry.ts
+import {
+  createServiceToken,
+  globalServiceRegistry,
+  registerInputResolver
+} from "@workglow/util";
+var KNOWLEDGE_BASES = createServiceToken("dataset.knowledge-bases");
+if (!globalServiceRegistry.has(KNOWLEDGE_BASES)) {
+  globalServiceRegistry.register(KNOWLEDGE_BASES, () => new Map, true);
+}
+function getGlobalKnowledgeBases() {
+  return globalServiceRegistry.get(KNOWLEDGE_BASES);
+}
+function registerKnowledgeBase(id, kb) {
+  const kbs = getGlobalKnowledgeBases();
+  kbs.set(id, kb);
+}
+function getKnowledgeBase(id) {
+  return getGlobalKnowledgeBases().get(id);
+}
+async function resolveKnowledgeBaseFromRegistry(id, format, registry) {
+  const kbs = registry.has(KNOWLEDGE_BASES) ? registry.get(KNOWLEDGE_BASES) : getGlobalKnowledgeBases();
+  const kb = kbs.get(id);
+  if (!kb) {
+    throw new Error(`Knowledge base "${id}" not found in registry`);
+  }
+  return kb;
+}
+registerInputResolver("dataset:knowledge-base", resolveKnowledgeBaseFromRegistry);
+// src/knowledge-base/createKnowledgeBase.ts
+import { InMemoryTabularStorage, InMemoryVectorStorage } from "@workglow/storage";
+
+// src/document/DocumentStorageSchema.ts
+var DocumentStorageSchema = {
   type: "object",
   properties: {
-    title: {
+    doc_id: {
       type: "string",
-      title: "Title",
-      description: "Document title"
+      "x-auto-generated": true,
+      title: "Document ID",
+      description: "Unique identifier for the document"
     },
-    sourceUri: {
+    data: {
       type: "string",
-      title: "Source URI",
-      description: "Original source URI of the document"
+      title: "Document Data",
+      description: "JSON-serialized document"
     },
-    createdAt: {
-      type: "string",
-      title: "Created At",
-      description: "ISO timestamp of creation"
+    metadata: {
+      type: "object",
+      title: "Metadata",
+      description: "Metadata of the document"
     }
   },
-  required: ["title"],
+  required: ["doc_id", "data"],
   additionalProperties: true
 };
+var DocumentStorageKey = ["doc_id"];
 
+// src/knowledge-base/createKnowledgeBase.ts
+async function createKnowledgeBase(options) {
+  const {
+    name,
+    vectorDimensions,
+    vectorType = Float32Array,
+    register: shouldRegister = true
+  } = options;
+  const tabularStorage = new InMemoryTabularStorage(DocumentStorageSchema, DocumentStorageKey);
+  await tabularStorage.setupDatabase();
+  const vectorStorage = new InMemoryVectorStorage(ChunkVectorStorageSchema, ChunkVectorPrimaryKey, [], vectorDimensions, vectorType);
+  await vectorStorage.setupDatabase();
+  const kb = new KnowledgeBase(name, tabularStorage, vectorStorage);
+  if (shouldRegister) {
+    registerKnowledgeBase(name, kb);
+  }
+  return kb;
+}
+// src/util/DatasetSchema.ts
+function TypeTabularStorage(options = {}) {
+  return {
+    title: "Tabular Storage",
+    description: "Storage ID or instance for tabular data storage",
+    ...options,
+    format: "storage:tabular",
+    oneOf: [
+      { type: "string", title: "Storage ID" },
+      { title: "Storage Instance", additionalProperties: true }
+    ]
+  };
+}
+function TypeKnowledgeBase(options = {}) {
+  return {
+    title: "Knowledge Base",
+    description: "Knowledge base ID or instance",
+    ...options,
+    format: "dataset:knowledge-base",
+    anyOf: [
+      { type: "string", title: "Knowledge Base ID" },
+      { title: "Knowledge Base Instance", additionalProperties: true }
+    ]
+  };
+}
 // src/document/DocumentNode.ts
 function estimateTokens(text) {
   return Math.ceil(text.length / 4);
@@ -725,31 +739,6 @@ function getDocumentRange(root, nodePath) {
   }
   return currentNode.range;
 }
-// src/document/DocumentStorageSchema.ts
-var DocumentStorageSchema = {
-  type: "object",
-  properties: {
-    doc_id: {
-      type: "string",
-      "x-auto-generated": true,
-      title: "Document ID",
-      description: "Unique identifier for the document"
-    },
-    data: {
-      type: "string",
-      title: "Document Data",
-      description: "JSON-serialized document"
-    },
-    metadata: {
-      type: "object",
-      title: "Metadata",
-      description: "Metadata of the document"
-    }
-  },
-  required: ["doc_id", "data"],
-  additionalProperties: true
-};
-var DocumentStorageKey = ["doc_id"];
 // src/document/StructuralParser.ts
 import { uuid4 } from "@workglow/util";
 class StructuralParser {
@@ -911,116 +900,19 @@ class StructuralParser {
     return /^#{1,6}\s/m.test(text);
   }
 }
-// src/document-chunk/DocumentChunkDataset.ts
-class DocumentChunkDataset {
-  storage;
-  constructor(storage) {
-    this.storage = storage;
-  }
-  getStorage() {
-    return this.storage;
-  }
-  async put(chunk) {
-    return this.storage.put(chunk);
-  }
-  async putBulk(chunks) {
-    return this.storage.putBulk(chunks);
-  }
-  async get(chunk_id) {
-    const key = { chunk_id };
-    return this.storage.get(key);
-  }
-  async delete(chunk_id) {
-    const key = { chunk_id };
-    return this.storage.delete(key);
-  }
-  async similaritySearch(query, options) {
-    return this.storage.similaritySearch(query, options);
-  }
-  async hybridSearch(query, options) {
-    if (this.storage.hybridSearch) {
-      return this.storage.hybridSearch(query, options);
-    }
-    throw new Error("Hybrid search not supported by this storage backend");
-  }
-  async getAll() {
-    return this.storage.getAll();
-  }
-  async size() {
-    return this.storage.size();
-  }
-  async clear() {
-    return this.storage.deleteAll();
-  }
-  destroy() {
-    return this.storage.destroy();
-  }
-  async setupDatabase() {
-    return this.storage.setupDatabase();
-  }
-  getVectorDimensions() {
-    return this.storage.getVectorDimensions();
-  }
-}
-// src/document-chunk/DocumentChunkDatasetRegistry.ts
-import {
-  createServiceToken as createServiceToken2,
-  globalServiceRegistry as globalServiceRegistry2,
-  registerInputResolver as registerInputResolver2
-} from "@workglow/util";
-var DOCUMENT_CHUNK_DATASET = createServiceToken2("dataset.document-chunk");
-if (!globalServiceRegistry2.has(DOCUMENT_CHUNK_DATASET)) {
-  globalServiceRegistry2.register(DOCUMENT_CHUNK_DATASET, () => new Map, true);
-}
-function getGlobalDocumentChunkDataset() {
-  return globalServiceRegistry2.get(DOCUMENT_CHUNK_DATASET);
-}
-function registerDocumentChunkDataset(id, dataset) {
-  const datasets = getGlobalDocumentChunkDataset();
-  datasets.set(id, dataset);
-}
-function getDocumentChunkDataset(id) {
-  return getGlobalDocumentChunkDataset().get(id);
-}
-async function resolveDocumentChunkDatasetFromRegistry(id, format, registry) {
-  const datasets = registry.has(DOCUMENT_CHUNK_DATASET) ? registry.get(DOCUMENT_CHUNK_DATASET) : getGlobalDocumentChunkDataset();
-  const dataset = datasets.get(id);
-  if (!dataset) {
-    throw new Error(`Document chunk dataset "${id}" not found in registry`);
-  }
-  return dataset;
-}
-registerInputResolver2("dataset:document-chunk", resolveDocumentChunkDatasetFromRegistry);
-// src/document-chunk/DocumentChunkSchema.ts
-import { TypedArraySchema } from "@workglow/util";
-var DocumentChunkSchema = {
-  type: "object",
-  properties: {
-    chunk_id: { type: "string", "x-auto-generated": true },
-    doc_id: { type: "string" },
-    vector: TypedArraySchema(),
-    metadata: { type: "object", format: "metadata", additionalProperties: true }
-  },
-  required: ["chunk_id", "doc_id", "vector", "metadata"],
-  additionalProperties: false
-};
-var DocumentChunkPrimaryKey = ["chunk_id"];
 export {
   traverseDepthFirst,
-  registerDocumentDataset,
-  registerDocumentChunkDataset,
+  registerKnowledgeBase,
   hasChildren,
   getNodePath,
-  getGlobalDocumentDatasets,
-  getGlobalDocumentChunkDataset,
+  getKnowledgeBase,
+  getGlobalKnowledgeBases,
   getDocumentRange,
-  getDocumentDataset,
-  getDocumentChunkDataset,
   getChildren,
   estimateTokens,
+  createKnowledgeBase,
   TypeTabularStorage,
-  TypeDocumentDataset,
-  TypeDocumentChunkDataset,
+  TypeKnowledgeBase,
   TopicNodeSchema,
   TokenBudgetSchema,
   StructuralParser,
@@ -1030,26 +922,20 @@ export {
   NodeRangeSchema,
   NodeKind,
   NodeEnrichmentSchema,
+  KnowledgeBase,
+  KNOWLEDGE_BASES,
   EntitySchema,
-  EnrichedChunkMetadataSchema,
-  EnrichedChunkMetadataArraySchema,
   DocumentStorageSchema,
   DocumentStorageKey,
   DocumentRootNodeSchema,
   DocumentNodeSchema,
   DocumentNodeBaseSchema,
   DocumentMetadataSchema,
-  DocumentDataset,
-  DocumentChunkSchema,
-  DocumentChunkPrimaryKey,
-  DocumentChunkDataset,
   Document,
-  DOCUMENT_DATASETS,
-  DOCUMENT_CHUNK_DATASET,
-  ChunkNodeSchema,
-  ChunkMetadataSchema,
-  ChunkMetadataArraySchema,
-  ChunkEnrichmentSchema
+  ChunkVectorStorageSchema,
+  ChunkVectorPrimaryKey,
+  ChunkRecordSchema,
+  ChunkRecordArraySchema
 };
 
-//# debugId=6A205B9D3594B93F64756E2164756E21
+//# debugId=22C0D4D10EDCEF7064756E2164756E21