@workglow/dataset 0.0.86

package/dist/bun.js

@@ -0,0 +1,1054 @@
+// @bun
+// src/util/DatasetSchema.ts
+function TypeTabularStorage(options = {}) {
+  return {
+    title: "Tabular Storage",
+    description: "Storage ID or instance for tabular data storage",
+    ...options,
+    format: "storage:tabular",
+    oneOf: [
+      { type: "string", title: "Storage ID" },
+      { title: "Storage Instance", additionalProperties: true }
+    ]
+  };
+}
+function TypeDocumentChunkDataset(options = {}) {
+  return {
+    title: "Document Chunk Dataset",
+    description: "Dataset ID or instance for document chunk data storage",
+    ...options,
+    format: "dataset:document-chunk",
+    anyOf: [
+      { type: "string", title: "Dataset ID" },
+      { title: "Dataset Instance", additionalProperties: true }
+    ]
+  };
+}
+function TypeDocumentDataset(options = {}) {
+  return {
+    title: "Document Dataset",
+    description: "Dataset ID or instance for document data storage",
+    ...options,
+    format: "dataset:document",
+    anyOf: [
+      { type: "string", title: "Dataset ID" },
+      { title: "Dataset Instance", additionalProperties: true }
+    ]
+  };
+}
+// src/document/Document.ts
+class Document {
+  doc_id;
+  metadata;
+  root;
+  chunks;
+  constructor(root, metadata, chunks = [], doc_id) {
+    this.doc_id = doc_id;
+    this.root = root;
+    this.metadata = metadata;
+    this.chunks = chunks || [];
+  }
+  setChunks(chunks) {
+    this.chunks = chunks;
+  }
+  getChunks() {
+    return this.chunks;
+  }
+  setDocId(doc_id) {
+    this.doc_id = doc_id;
+  }
+  findChunksByNodeId(nodeId) {
+    return this.chunks.filter((chunk) => chunk.nodePath.includes(nodeId));
+  }
+  toJSON() {
+    return {
+      metadata: this.metadata,
+      root: this.root,
+      chunks: this.chunks
+    };
+  }
+  static fromJSON(json, doc_id) {
+    const obj = JSON.parse(json);
+    return new Document(obj.root, obj.metadata, obj.chunks, doc_id);
+  }
+}
+// src/document/DocumentDataset.ts
+class DocumentDataset {
+  tabularStorage;
+  vectorStorage;
+  constructor(tabularStorage, vectorStorage) {
+    this.tabularStorage = tabularStorage;
+    this.vectorStorage = vectorStorage;
+  }
+  async upsert(document) {
+    const serialized = JSON.stringify(document.toJSON());
+    const insertEntity = {
+      doc_id: document.doc_id,
+      data: serialized
+    };
+    const entity = await this.tabularStorage.put(insertEntity);
+    if (document.doc_id !== entity.doc_id) {
+      document.setDocId(entity.doc_id);
+    }
+    return document;
+  }
+  async get(doc_id) {
+    const entity = await this.tabularStorage.get({ doc_id });
+    if (!entity) {
+      return;
+    }
+    return Document.fromJSON(entity.data, entity.doc_id);
+  }
+  async delete(doc_id) {
+    await this.tabularStorage.delete({ doc_id });
+  }
+  async getNode(doc_id, nodeId) {
+    const doc = await this.get(doc_id);
+    if (!doc) {
+      return;
+    }
+    const traverse = (node) => {
+      if (node.nodeId === nodeId) {
+        return node;
+      }
+      if (node.children && Array.isArray(node.children)) {
+        for (const child of node.children) {
+          const found = traverse(child);
+          if (found)
+            return found;
+        }
+      }
+      return;
+    };
+    return traverse(doc.root);
+  }
+  async getAncestors(doc_id, nodeId) {
+    const doc = await this.get(doc_id);
+    if (!doc) {
+      return [];
+    }
+    const path = [];
+    const findPath = (node) => {
+      path.push(node.nodeId);
+      if (node.nodeId === nodeId) {
+        return true;
+      }
+      if (node.children && Array.isArray(node.children)) {
+        for (const child of node.children) {
+          if (findPath(child)) {
+            return true;
+          }
+        }
+      }
+      path.pop();
+      return false;
+    };
+    if (!findPath(doc.root)) {
+      return [];
+    }
+    const ancestors = [];
+    let currentNode = doc.root;
+    ancestors.push(currentNode);
+    for (let i = 1;i < path.length; i++) {
+      const targetId = path[i];
+      if (currentNode.children && Array.isArray(currentNode.children)) {
+        const found = currentNode.children.find((child) => child.nodeId === targetId);
+        if (found) {
+          currentNode = found;
+          ancestors.push(currentNode);
+        } else {
+          break;
+        }
+      } else {
+        break;
+      }
+    }
+    return ancestors;
+  }
+  async getChunks(doc_id) {
+    const doc = await this.get(doc_id);
+    if (!doc) {
+      return [];
+    }
+    return doc.getChunks();
+  }
+  async findChunksByNodeId(doc_id, nodeId) {
+    const doc = await this.get(doc_id);
+    if (!doc) {
+      return [];
+    }
+    if (doc.findChunksByNodeId) {
+      return doc.findChunksByNodeId(nodeId);
+    }
+    const chunks = doc.getChunks();
+    return chunks.filter((chunk) => chunk.nodePath && chunk.nodePath.includes(nodeId));
+  }
+  async list() {
+    const entities = await this.tabularStorage.getAll();
+    if (!entities) {
+      return [];
+    }
+    return entities.map((e) => e.doc_id);
+  }
+  async search(query, options) {
+    return this.vectorStorage?.similaritySearch(query, options) || [];
+  }
+}
+// src/document/DocumentDatasetRegistry.ts
+import {
+  createServiceToken,
+  globalServiceRegistry,
+  registerInputResolver
+} from "@workglow/util";
+var DOCUMENT_DATASETS = createServiceToken("dataset.documents");
+if (!globalServiceRegistry.has(DOCUMENT_DATASETS)) {
+  globalServiceRegistry.register(DOCUMENT_DATASETS, () => new Map, true);
+}
+function getGlobalDocumentDatasets() {
+  return globalServiceRegistry.get(DOCUMENT_DATASETS);
+}
+function registerDocumentDataset(id, dataset) {
+  const datasets = getGlobalDocumentDatasets();
+  datasets.set(id, dataset);
+}
+function getDocumentDataset(id) {
+  return getGlobalDocumentDatasets().get(id);
+}
+async function resolveDocumentDatasetFromRegistry(id, format, registry) {
+  const datasets = registry.has(DOCUMENT_DATASETS) ? registry.get(DOCUMENT_DATASETS) : getGlobalDocumentDatasets();
+  const dataset = datasets.get(id);
+  if (!dataset) {
+    throw new Error(`Document dataset "${id}" not found in registry`);
+  }
+  return dataset;
+}
+registerInputResolver("dataset:document", resolveDocumentDatasetFromRegistry);
+// src/document/DocumentSchema.ts
+var NodeKind = {
+  DOCUMENT: "document",
+  SECTION: "section",
+  PARAGRAPH: "paragraph",
+  SENTENCE: "sentence",
+  TOPIC: "topic"
+};
+var NodeRangeSchema = {
+  type: "object",
+  properties: {
+    startOffset: {
+      type: "integer",
+      title: "Start Offset",
+      description: "Starting character offset"
+    },
+    endOffset: {
+      type: "integer",
+      title: "End Offset",
+      description: "Ending character offset"
+    }
+  },
+  required: ["startOffset", "endOffset"],
+  additionalProperties: false
+};
+var EntitySchema = {
+  type: "object",
+  properties: {
+    text: {
+      type: "string",
+      title: "Text",
+      description: "Entity text"
+    },
+    type: {
+      type: "string",
+      title: "Type",
+      description: "Entity type (e.g., PERSON, ORG, LOC)"
+    },
+    score: {
+      type: "number",
+      title: "Score",
+      description: "Confidence score"
+    }
+  },
+  required: ["text", "type", "score"],
+  additionalProperties: false
+};
+var NodeEnrichmentSchema = {
+  type: "object",
+  properties: {
+    summary: {
+      type: "string",
+      title: "Summary",
+      description: "Summary of the node content"
+    },
+    entities: {
+      type: "array",
+      items: EntitySchema,
+      title: "Entities",
+      description: "Named entities extracted from the node"
+    },
+    keywords: {
+      type: "array",
+      items: { type: "string" },
+      title: "Keywords",
+      description: "Keywords associated with the node"
+    }
+  },
+  additionalProperties: false
+};
+var DocumentNodeBaseSchema = {
+  type: "object",
+  properties: {
+    nodeId: {
+      type: "string",
+      title: "Node ID",
+      description: "Unique identifier for this node"
+    },
+    kind: {
+      type: "string",
+      enum: Object.values(NodeKind),
+      title: "Kind",
+      description: "Node type discriminator"
+    },
+    range: NodeRangeSchema,
+    text: {
+      type: "string",
+      title: "Text",
+      description: "Text content of the node"
+    },
+    enrichment: NodeEnrichmentSchema
+  },
+  required: ["nodeId", "kind", "range", "text"],
+  additionalProperties: true
+};
+var DocumentNodeSchema = {
+  type: "object",
+  title: "Document Node",
+  description: "A node in the hierarchical document tree",
+  properties: {
+    ...DocumentNodeBaseSchema.properties,
+    level: {
+      type: "integer",
+      title: "Level",
+      description: "Header level for section nodes"
+    },
+    title: {
+      type: "string",
+      title: "Title",
+      description: "Section title"
+    },
+    children: {
+      type: "array",
+      title: "Children",
+      description: "Child nodes"
+    }
+  },
+  required: [...DocumentNodeBaseSchema.required],
+  additionalProperties: false
+};
+var ParagraphNodeSchema = {
+  type: "object",
+  properties: {
+    ...DocumentNodeBaseSchema.properties,
+    kind: {
+      type: "string",
+      const: NodeKind.PARAGRAPH,
+      title: "Kind",
+      description: "Node type discriminator"
+    }
+  },
+  required: [...DocumentNodeBaseSchema.required],
+  additionalProperties: false
+};
+var SentenceNodeSchema = {
+  type: "object",
+  properties: {
+    ...DocumentNodeBaseSchema.properties,
+    kind: {
+      type: "string",
+      const: NodeKind.SENTENCE,
+      title: "Kind",
+      description: "Node type discriminator"
+    }
+  },
+  required: [...DocumentNodeBaseSchema.required],
+  additionalProperties: false
+};
+var SectionNodeSchema = {
+  type: "object",
+  properties: {
+    ...DocumentNodeBaseSchema.properties,
+    kind: {
+      type: "string",
+      const: NodeKind.SECTION,
+      title: "Kind",
+      description: "Node type discriminator"
+    },
+    level: {
+      type: "integer",
+      minimum: 1,
+      maximum: 6,
+      title: "Level",
+      description: "Header level (1-6 for markdown)"
+    },
+    title: {
+      type: "string",
+      title: "Title",
+      description: "Section title"
+    },
+    children: {
+      type: "array",
+      items: DocumentNodeSchema,
+      title: "Children",
+      description: "Child nodes"
+    }
+  },
+  required: [...DocumentNodeBaseSchema.required, "level", "title", "children"],
+  additionalProperties: false
+};
+var TopicNodeSchema = {
+  type: "object",
+  properties: {
+    ...DocumentNodeBaseSchema.properties,
+    kind: {
+      type: "string",
+      const: NodeKind.TOPIC,
+      title: "Kind",
+      description: "Node type discriminator"
+    },
+    children: {
+      type: "array",
+      items: DocumentNodeSchema,
+      title: "Children",
+      description: "Child nodes"
+    }
+  },
+  required: [...DocumentNodeBaseSchema.required, "children"],
+  additionalProperties: false
+};
+var DocumentRootNodeSchema = {
+  type: "object",
+  properties: {
+    ...DocumentNodeBaseSchema.properties,
+    kind: {
+      type: "string",
+      const: NodeKind.DOCUMENT,
+      title: "Kind",
+      description: "Node type discriminator"
+    },
+    title: {
+      type: "string",
+      title: "Title",
+      description: "Document title"
+    },
+    children: {
+      type: "array",
+      items: DocumentNodeSchema,
+      title: "Children",
+      description: "Child nodes"
+    }
+  },
+  required: [...DocumentNodeBaseSchema.required, "title", "children"],
+  additionalProperties: false
+};
+var TokenBudgetSchema = {
+  type: "object",
+  properties: {
+    maxTokensPerChunk: {
+      type: "integer",
+      title: "Max Tokens Per Chunk",
+      description: "Maximum tokens allowed per chunk"
+    },
+    overlapTokens: {
+      type: "integer",
+      title: "Overlap Tokens",
+      description: "Number of tokens to overlap between chunks"
+    },
+    reservedTokens: {
+      type: "integer",
+      title: "Reserved Tokens",
+      description: "Tokens reserved for metadata or context"
+    }
+  },
+  required: ["maxTokensPerChunk", "overlapTokens", "reservedTokens"],
+  additionalProperties: false
+};
+var ChunkEnrichmentSchema = {
+  type: "object",
+  properties: {
+    summary: {
+      type: "string",
+      title: "Summary",
+      description: "Summary of the chunk content"
+    },
+    entities: {
+      type: "array",
+      items: EntitySchema,
+      title: "Entities",
+      description: "Named entities extracted from the chunk"
+    }
+  },
+  additionalProperties: false
+};
+var ChunkNodeSchema = () => ({
+  type: "object",
+  properties: {
+    chunkId: {
+      type: "string",
+      title: "Chunk ID",
+      description: "Unique identifier for this chunk"
+    },
+    doc_id: {
+      type: "string",
+      title: "Document ID",
+      description: "ID of the parent document"
+    },
+    text: {
+      type: "string",
+      title: "Text",
+      description: "Text content of the chunk"
+    },
+    nodePath: {
+      type: "array",
+      items: { type: "string" },
+      title: "Node Path",
+      description: "Node IDs from root to leaf"
+    },
+    depth: {
+      type: "integer",
+      title: "Depth",
+      description: "Depth in the document tree"
+    },
+    enrichment: ChunkEnrichmentSchema
+  },
+  required: ["chunkId", "doc_id", "text", "nodePath", "depth"],
+  additionalProperties: false
+});
+var ChunkMetadataSchema = {
+  type: "object",
+  properties: {
+    doc_id: {
+      type: "string",
+      title: "Document ID",
+      description: "ID of the parent document"
+    },
+    chunkId: {
+      type: "string",
+      title: "Chunk ID",
+      description: "Unique identifier for this chunk"
+    },
+    leafNodeId: {
+      type: "string",
+      title: "Leaf Node ID",
+      description: "ID of the leaf node this chunk belongs to"
+    },
+    depth: {
+      type: "integer",
+      title: "Depth",
+      description: "Depth in the document tree"
+    },
+    text: {
+      type: "string",
+      title: "Text",
+      description: "Text content of the chunk"
+    },
+    nodePath: {
+      type: "array",
+      items: { type: "string" },
+      title: "Node Path",
+      description: "Node IDs from root to leaf"
+    },
+    summary: {
+      type: "string",
+      title: "Summary",
+      description: "Summary of the chunk content"
+    },
+    entities: {
+      type: "array",
+      items: EntitySchema,
+      title: "Entities",
+      description: "Named entities extracted from the chunk"
+    }
+  },
+  required: ["doc_id", "chunkId", "leafNodeId", "depth", "text", "nodePath"],
+  additionalProperties: true
+};
+var ChunkMetadataArraySchema = {
+  type: "array",
+  items: ChunkMetadataSchema,
+  title: "Chunk Metadata",
+  description: "Metadata for each chunk"
+};
+var EnrichedChunkMetadataSchema = {
+  type: "object",
+  properties: {
+    doc_id: {
+      type: "string",
+      title: "Document ID",
+      description: "ID of the parent document"
+    },
+    chunkId: {
+      type: "string",
+      title: "Chunk ID",
+      description: "Unique identifier for this chunk"
+    },
+    leafNodeId: {
+      type: "string",
+      title: "Leaf Node ID",
+      description: "ID of the leaf node this chunk belongs to"
+    },
+    depth: {
+      type: "integer",
+      title: "Depth",
+      description: "Depth in the document tree"
+    },
+    text: {
+      type: "string",
+      title: "Text",
+      description: "Text content of the chunk"
+    },
+    nodePath: {
+      type: "array",
+      items: { type: "string" },
+      title: "Node Path",
+      description: "Node IDs from root to leaf"
+    },
+    summary: {
+      type: "string",
+      title: "Summary",
+      description: "Summary of the chunk content"
+    },
+    entities: {
+      type: "array",
+      items: EntitySchema,
+      title: "Entities",
+      description: "Named entities (rolled up from hierarchy)"
+    },
+    parentSummaries: {
+      type: "array",
+      items: { type: "string" },
+      title: "Parent Summaries",
+      description: "Summaries from ancestor nodes"
+    },
+    sectionTitles: {
+      type: "array",
+      items: { type: "string" },
+      title: "Section Titles",
+      description: "Titles of ancestor section nodes"
+    }
+  },
+  required: ["doc_id", "chunkId", "leafNodeId", "depth", "text", "nodePath"],
+  additionalProperties: true
+};
+var EnrichedChunkMetadataArraySchema = {
+  type: "array",
+  items: EnrichedChunkMetadataSchema,
+  title: "Enriched Metadata",
+  description: "Metadata enriched with hierarchy information"
+};
+var DocumentMetadataSchema = {
+  type: "object",
+  properties: {
+    title: {
+      type: "string",
+      title: "Title",
+      description: "Document title"
+    },
+    sourceUri: {
+      type: "string",
+      title: "Source URI",
+      description: "Original source URI of the document"
+    },
+    createdAt: {
+      type: "string",
+      title: "Created At",
+      description: "ISO timestamp of creation"
+    }
+  },
+  required: ["title"],
+  additionalProperties: true
+};
+
+// src/document/DocumentNode.ts
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function hasChildren(node) {
+  return node.kind === NodeKind.DOCUMENT || node.kind === NodeKind.SECTION || node.kind === NodeKind.TOPIC;
+}
+function getChildren(node) {
+  if (hasChildren(node)) {
+    return node.children;
+  }
+  return [];
+}
+function* traverseDepthFirst(node) {
+  yield node;
+  if (hasChildren(node)) {
+    for (const child of node.children) {
+      yield* traverseDepthFirst(child);
+    }
+  }
+}
+function getNodePath(root, targetNodeId) {
+  const path = [];
+  function search(node) {
+    path.push(node.nodeId);
+    if (node.nodeId === targetNodeId) {
+      return true;
+    }
+    if (hasChildren(node)) {
+      for (const child of node.children) {
+        if (search(child)) {
+          return true;
+        }
+      }
+    }
+    path.pop();
+    return false;
+  }
+  return search(root) ? path : undefined;
+}
+function getDocumentRange(root, nodePath) {
+  let currentNode = root;
+  for (let i = 1;i < nodePath.length; i++) {
+    const targetId = nodePath[i];
+    const children = currentNode.children;
+    let found;
+    for (let j = 0;j < children.length; j++) {
+      if (children[j].nodeId === targetId) {
+        found = children[j];
+        break;
+      }
+    }
+    if (!found) {
+      throw new Error(`Node with id ${targetId} not found in path`);
+    }
+    currentNode = found;
+  }
+  return currentNode.range;
+}
+// src/document/DocumentStorageSchema.ts
+var DocumentStorageSchema = {
+  type: "object",
+  properties: {
+    doc_id: {
+      type: "string",
+      "x-auto-generated": true,
+      title: "Document ID",
+      description: "Unique identifier for the document"
+    },
+    data: {
+      type: "string",
+      title: "Document Data",
+      description: "JSON-serialized document"
+    },
+    metadata: {
+      type: "object",
+      title: "Metadata",
+      description: "Metadata of the document"
+    }
+  },
+  required: ["doc_id", "data"],
+  additionalProperties: true
+};
+var DocumentStorageKey = ["doc_id"];
+// src/document/StructuralParser.ts
+import { uuid4 } from "@workglow/util";
+class StructuralParser {
+  static async parseMarkdown(doc_id, text, title) {
+    const lines = text.split(`
+`);
+    let currentOffset = 0;
+    const root = {
+      nodeId: uuid4(),
+      kind: NodeKind.DOCUMENT,
+      range: { startOffset: 0, endOffset: text.length },
+      text: title,
+      title,
+      children: []
+    };
+    let currentParentStack = [root];
+    let textBuffer = [];
+    let textBufferStartOffset = 0;
+    const flushTextBuffer = async () => {
+      if (textBuffer.length > 0) {
+        const content = textBuffer.join(`
+`).trim();
+        if (content) {
+          const paragraphStartOffset = textBufferStartOffset;
+          const paragraphEndOffset = currentOffset;
+          const paragraph = {
+            nodeId: uuid4(),
+            kind: NodeKind.PARAGRAPH,
+            range: {
+              startOffset: paragraphStartOffset,
+              endOffset: paragraphEndOffset
+            },
+            text: content
+          };
+          currentParentStack[currentParentStack.length - 1].children.push(paragraph);
+        }
+        textBuffer = [];
+      }
+    };
+    for (const line of lines) {
+      const lineLength = line.length + 1;
+      const headerMatch = line.match(/^(#{1,6})\s+(.*)$/);
+      if (headerMatch) {
+        await flushTextBuffer();
+        const level = headerMatch[1].length;
+        const headerTitle = headerMatch[2];
+        while (currentParentStack.length > 1 && currentParentStack[currentParentStack.length - 1].kind === NodeKind.SECTION && currentParentStack[currentParentStack.length - 1].level >= level) {
+          const poppedSection = currentParentStack.pop();
+          const updatedSection = {
+            ...poppedSection,
+            range: {
+              ...poppedSection.range,
+              endOffset: currentOffset
+            }
+          };
+          const parent = currentParentStack[currentParentStack.length - 1];
+          parent.children[parent.children.length - 1] = updatedSection;
+        }
+        const sectionStartOffset = currentOffset;
+        const section = {
+          nodeId: uuid4(),
+          kind: NodeKind.SECTION,
+          level,
+          title: headerTitle,
+          range: {
+            startOffset: sectionStartOffset,
+            endOffset: text.length
+          },
+          text: headerTitle,
+          children: []
+        };
+        currentParentStack[currentParentStack.length - 1].children.push(section);
+        currentParentStack.push(section);
+      } else {
+        if (textBuffer.length === 0) {
+          textBufferStartOffset = currentOffset;
+        }
+        textBuffer.push(line);
+      }
+      currentOffset += lineLength;
+    }
+    await flushTextBuffer();
+    while (currentParentStack.length > 1) {
+      const section = currentParentStack.pop();
+      const updatedSection = {
+        ...section,
+        range: {
+          ...section.range,
+          endOffset: text.length
+        }
+      };
+      const parent = currentParentStack[currentParentStack.length - 1];
+      parent.children[parent.children.length - 1] = updatedSection;
+    }
+    return root;
+  }
+  static async parsePlainText(doc_id, text, title) {
+    const root = {
+      nodeId: uuid4(),
+      kind: NodeKind.DOCUMENT,
+      range: { startOffset: 0, endOffset: text.length },
+      text: title,
+      title,
+      children: []
+    };
+    const paragraphRegex = /\n\s*\n/g;
+    let lastIndex = 0;
+    let paragraphIndex = 0;
+    let match;
+    while ((match = paragraphRegex.exec(text)) !== null) {
+      const rawParagraph = text.slice(lastIndex, match.index);
+      const paragraphText = rawParagraph.trim();
+      if (paragraphText.length > 0) {
+        const trimmedRelativeStart = rawParagraph.indexOf(paragraphText);
+        const startOffset = lastIndex + trimmedRelativeStart;
+        const endOffset = startOffset + paragraphText.length;
+        const paragraph = {
+          nodeId: uuid4(),
+          kind: NodeKind.PARAGRAPH,
+          range: {
+            startOffset,
+            endOffset
+          },
+          text: paragraphText
+        };
+        root.children.push(paragraph);
+        paragraphIndex++;
+      }
+      lastIndex = paragraphRegex.lastIndex;
+    }
+    if (lastIndex < text.length) {
+      const rawParagraph = text.slice(lastIndex);
+      const paragraphText = rawParagraph.trim();
+      if (paragraphText.length > 0) {
+        const trimmedRelativeStart = rawParagraph.indexOf(paragraphText);
+        const startOffset = lastIndex + trimmedRelativeStart;
+        const endOffset = startOffset + paragraphText.length;
+        const paragraph = {
+          nodeId: uuid4(),
+          kind: NodeKind.PARAGRAPH,
+          range: {
+            startOffset,
+            endOffset
+          },
+          text: paragraphText
+        };
+        root.children.push(paragraph);
+      }
+    }
+    return root;
+  }
+  static parse(doc_id, text, title, format) {
+    if (format === "markdown" || !format && this.looksLikeMarkdown(text)) {
+      return this.parseMarkdown(doc_id, text, title);
+    }
+    return this.parsePlainText(doc_id, text, title);
+  }
+  static looksLikeMarkdown(text) {
+    return /^#{1,6}\s/m.test(text);
+  }
+}
+// src/document-chunk/DocumentChunkDataset.ts
+class DocumentChunkDataset {
+  storage;
+  constructor(storage) {
+    this.storage = storage;
+  }
+  getStorage() {
+    return this.storage;
+  }
+  async put(chunk) {
+    return this.storage.put(chunk);
+  }
+  async putBulk(chunks) {
+    return this.storage.putBulk(chunks);
+  }
+  async get(chunk_id) {
+    const key = { chunk_id };
+    return this.storage.get(key);
+  }
+  async delete(chunk_id) {
+    const key = { chunk_id };
+    return this.storage.delete(key);
+  }
+  async similaritySearch(query, options) {
+    return this.storage.similaritySearch(query, options);
+  }
+  async hybridSearch(query, options) {
+    if (this.storage.hybridSearch) {
+      return this.storage.hybridSearch(query, options);
+    }
+    throw new Error("Hybrid search not supported by this storage backend");
+  }
+  async getAll() {
+    return this.storage.getAll();
+  }
+  async size() {
+    return this.storage.size();
+  }
+  async clear() {
+    return this.storage.clear();
+  }
+  destroy() {
+    return this.storage.destroy();
+  }
+  async setupDatabase() {
+    return this.storage.setupDatabase();
+  }
+  getVectorDimensions() {
+    return this.storage.getVectorDimensions();
+  }
+}
+// src/document-chunk/DocumentChunkDatasetRegistry.ts
+import {
+  createServiceToken as createServiceToken2,
+  globalServiceRegistry as globalServiceRegistry2,
+  registerInputResolver as registerInputResolver2
+} from "@workglow/util";
+var DOCUMENT_CHUNK_DATASET = createServiceToken2("dataset.document-chunk");
+if (!globalServiceRegistry2.has(DOCUMENT_CHUNK_DATASET)) {
+  globalServiceRegistry2.register(DOCUMENT_CHUNK_DATASET, () => new Map, true);
+}
+function getGlobalDocumentChunkDataset() {
+  return globalServiceRegistry2.get(DOCUMENT_CHUNK_DATASET);
+}
+function registerDocumentChunkDataset(id, dataset) {
+  const datasets = getGlobalDocumentChunkDataset();
+  datasets.set(id, dataset);
+}
+function getDocumentChunkDataset(id) {
+  return getGlobalDocumentChunkDataset().get(id);
+}
+async function resolveDocumentChunkDatasetFromRegistry(id, format, registry) {
+  const datasets = registry.has(DOCUMENT_CHUNK_DATASET) ? registry.get(DOCUMENT_CHUNK_DATASET) : getGlobalDocumentChunkDataset();
+  const dataset = datasets.get(id);
+  if (!dataset) {
+    throw new Error(`Document chunk dataset "${id}" not found in registry`);
+  }
+  return dataset;
+}
+registerInputResolver2("dataset:document-chunk", resolveDocumentChunkDatasetFromRegistry);
+// src/document-chunk/DocumentChunkSchema.ts
+import { TypedArraySchema } from "@workglow/util";
+var DocumentChunkSchema = {
+  type: "object",
+  properties: {
+    chunk_id: { type: "string", "x-auto-generated": true },
+    doc_id: { type: "string" },
+    vector: TypedArraySchema(),
+    metadata: { type: "object", format: "metadata", additionalProperties: true }
+  },
+  additionalProperties: false
+};
+var DocumentChunkPrimaryKey = ["chunk_id"];
+export {
+  traverseDepthFirst,
+  registerDocumentDataset,
+  registerDocumentChunkDataset,
+  hasChildren,
+  getNodePath,
+  getGlobalDocumentDatasets,
+  getGlobalDocumentChunkDataset,
+  getDocumentRange,
+  getDocumentDataset,
+  getDocumentChunkDataset,
+  getChildren,
+  estimateTokens,
+  TypeTabularStorage,
+  TypeDocumentDataset,
+  TypeDocumentChunkDataset,
+  TopicNodeSchema,
+  TokenBudgetSchema,
+  StructuralParser,
+  SentenceNodeSchema,
+  SectionNodeSchema,
+  ParagraphNodeSchema,
+  NodeRangeSchema,
+  NodeKind,
+  NodeEnrichmentSchema,
+  EntitySchema,
+  EnrichedChunkMetadataSchema,
+  EnrichedChunkMetadataArraySchema,
+  DocumentStorageSchema,
+  DocumentStorageKey,
+  DocumentRootNodeSchema,
+  DocumentNodeSchema,
+  DocumentNodeBaseSchema,
+  DocumentMetadataSchema,
+  DocumentDataset,
+  DocumentChunkSchema,
+  DocumentChunkPrimaryKey,
+  DocumentChunkDataset,
+  Document,
+  DOCUMENT_DATASETS,
+  DOCUMENT_CHUNK_DATASET,
+  ChunkNodeSchema,
+  ChunkMetadataSchema,
+  ChunkMetadataArraySchema,
+  ChunkEnrichmentSchema
+};
+
+//# debugId=1AB5FF88396B3AD064756E2164756E21