@workglow/dataset 0.0.110 → 0.0.114

package/dist/chunk/ChunkSchema.d.ts

@@ -0,0 +1,206 @@
+/**
+ * @license
+ * Copyright 2025 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import type { FromSchema } from "@workglow/util";
+/**
+ * Schema for a unified chunk record.
+ * Replaces ChunkNode, ChunkMetadata, and EnrichedChunkMetadata with a single flat type.
+ */
+export declare const ChunkRecordSchema: () => {
+    readonly type: "object";
+    readonly properties: {
+        readonly chunkId: {
+            readonly type: "string";
+            readonly title: "Chunk ID";
+            readonly description: "Unique identifier for this chunk";
+        };
+        readonly doc_id: {
+            readonly type: "string";
+            readonly title: "Document ID";
+            readonly description: "ID of the parent document";
+        };
+        readonly text: {
+            readonly type: "string";
+            readonly title: "Text";
+            readonly description: "Text content of the chunk";
+        };
+        readonly nodePath: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "string";
+            };
+            readonly title: "Node Path";
+            readonly description: "Node IDs from root to leaf";
+        };
+        readonly depth: {
+            readonly type: "integer";
+            readonly title: "Depth";
+            readonly description: "Depth in the document tree";
+        };
+        readonly leafNodeId: {
+            readonly type: "string";
+            readonly title: "Leaf Node ID";
+            readonly description: "ID of the leaf node this chunk belongs to";
+        };
+        readonly summary: {
+            readonly type: "string";
+            readonly title: "Summary";
+            readonly description: "Summary of the chunk content";
+        };
+        readonly entities: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "object";
+                readonly properties: {
+                    readonly text: {
+                        readonly type: "string";
+                        readonly title: "Text";
+                        readonly description: "Entity text";
+                    };
+                    readonly type: {
+                        readonly type: "string";
+                        readonly title: "Type";
+                        readonly description: "Entity type (e.g., PERSON, ORG, LOC)";
+                    };
+                    readonly score: {
+                        readonly type: "number";
+                        readonly title: "Score";
+                        readonly description: "Confidence score";
+                    };
+                };
+                readonly required: readonly ["text", "type", "score"];
+                readonly additionalProperties: false;
+            };
+            readonly title: "Entities";
+            readonly description: "Named entities extracted from the chunk";
+        };
+        readonly parentSummaries: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "string";
+            };
+            readonly title: "Parent Summaries";
+            readonly description: "Summaries from ancestor nodes";
+        };
+        readonly sectionTitles: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "string";
+            };
+            readonly title: "Section Titles";
+            readonly description: "Titles of ancestor section nodes";
+        };
+        readonly doc_title: {
+            readonly type: "string";
+            readonly title: "Document Title";
+            readonly description: "Title of the parent document";
+        };
+    };
+    readonly required: readonly ["chunkId", "doc_id", "text", "nodePath", "depth"];
+    readonly additionalProperties: true;
+};
+export type ChunkRecord = FromSchema<ReturnType<typeof ChunkRecordSchema>>;
+/**
+ * Schema for chunk record array (for use in task schemas)
+ */
+export declare const ChunkRecordArraySchema: {
+    readonly type: "array";
+    readonly items: {
+        readonly type: "object";
+        readonly properties: {
+            readonly chunkId: {
+                readonly type: "string";
+                readonly title: "Chunk ID";
+                readonly description: "Unique identifier for this chunk";
+            };
+            readonly doc_id: {
+                readonly type: "string";
+                readonly title: "Document ID";
+                readonly description: "ID of the parent document";
+            };
+            readonly text: {
+                readonly type: "string";
+                readonly title: "Text";
+                readonly description: "Text content of the chunk";
+            };
+            readonly nodePath: {
+                readonly type: "array";
+                readonly items: {
+                    readonly type: "string";
+                };
+                readonly title: "Node Path";
+                readonly description: "Node IDs from root to leaf";
+            };
+            readonly depth: {
+                readonly type: "integer";
+                readonly title: "Depth";
+                readonly description: "Depth in the document tree";
+            };
+            readonly leafNodeId: {
+                readonly type: "string";
+                readonly title: "Leaf Node ID";
+                readonly description: "ID of the leaf node this chunk belongs to";
+            };
+            readonly summary: {
+                readonly type: "string";
+                readonly title: "Summary";
+                readonly description: "Summary of the chunk content";
+            };
+            readonly entities: {
+                readonly type: "array";
+                readonly items: {
+                    readonly type: "object";
+                    readonly properties: {
+                        readonly text: {
+                            readonly type: "string";
+                            readonly title: "Text";
+                            readonly description: "Entity text";
+                        };
+                        readonly type: {
+                            readonly type: "string";
+                            readonly title: "Type";
+                            readonly description: "Entity type (e.g., PERSON, ORG, LOC)";
+                        };
+                        readonly score: {
+                            readonly type: "number";
+                            readonly title: "Score";
+                            readonly description: "Confidence score";
+                        };
+                    };
+                    readonly required: readonly ["text", "type", "score"];
+                    readonly additionalProperties: false;
+                };
+                readonly title: "Entities";
+                readonly description: "Named entities extracted from the chunk";
+            };
+            readonly parentSummaries: {
+                readonly type: "array";
+                readonly items: {
+                    readonly type: "string";
+                };
+                readonly title: "Parent Summaries";
+                readonly description: "Summaries from ancestor nodes";
+            };
+            readonly sectionTitles: {
+                readonly type: "array";
+                readonly items: {
+                    readonly type: "string";
+                };
+                readonly title: "Section Titles";
+                readonly description: "Titles of ancestor section nodes";
+            };
+            readonly doc_title: {
+                readonly type: "string";
+                readonly title: "Document Title";
+                readonly description: "Title of the parent document";
+            };
+        };
+        readonly required: readonly ["chunkId", "doc_id", "text", "nodePath", "depth"];
+        readonly additionalProperties: true;
+    };
+    readonly title: "Chunk Records";
+    readonly description: "Array of chunk records";
+};
+//# sourceMappingURL=ChunkSchema.d.ts.map

package/dist/chunk/ChunkSchema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChunkSchema.d.ts","sourceRoot":"","sources":["../../src/chunk/ChunkSchema.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAkB,UAAU,EAAc,MAAM,gBAAgB,CAAC;AAG7E;;;GAGG;AACH,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkEQ,CAAC;AAEvC,MAAM,MAAM,WAAW,GAAG,UAAU,CAAC,UAAU,CAAC,OAAO,iBAAiB,CAAC,CAAC,CAAC;AAE3E;;GAEG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAKJ,CAAC"}

package/dist/chunk/ChunkVectorStorageSchema.d.ts

@@ -0,0 +1,64 @@
+/**
+ * @license
+ * Copyright 2025 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { IVectorStorage } from "@workglow/storage";
+import { type TypedArray } from "@workglow/util";
+import type { ChunkRecord } from "./ChunkSchema";
+/**
+ * Schema for chunk vector storage with typed metadata.
+ * Replaces DocumentChunkSchema with ChunkRecord as the metadata type.
+ */
+export declare const ChunkVectorStorageSchema: {
+    readonly type: "object";
+    readonly properties: {
+        readonly chunk_id: {
+            readonly type: "string";
+            readonly "x-auto-generated": true;
+        };
+        readonly doc_id: {
+            readonly type: "string";
+        };
+        readonly vector: {
+            readonly type: "array";
+            readonly format: "TypedArray";
+            readonly title: "Typed Array";
+            readonly description: "A typed array (Float32Array, Int8Array, etc.)";
+        };
+        readonly metadata: {
+            readonly type: "object";
+            readonly format: "metadata";
+            readonly additionalProperties: true;
+        };
+    };
+    readonly required: readonly ["chunk_id", "doc_id", "vector", "metadata"];
+    readonly additionalProperties: false;
+};
+export type ChunkVectorStorageSchema = typeof ChunkVectorStorageSchema;
+export declare const ChunkVectorPrimaryKey: readonly ["chunk_id"];
+export type ChunkVectorPrimaryKey = typeof ChunkVectorPrimaryKey;
+export interface ChunkVectorEntity<Metadata extends ChunkRecord = ChunkRecord, Vector extends TypedArray = TypedArray> {
+    chunk_id: string;
+    doc_id: string;
+    vector: Vector;
+    metadata: Metadata;
+}
+/**
+ * Type for inserting chunk vectors - chunk_id is optional (auto-generated)
+ */
+export type InsertChunkVectorEntity<Metadata extends ChunkRecord = ChunkRecord, Vector extends TypedArray = TypedArray> = Omit<ChunkVectorEntity<Metadata, Vector>, "chunk_id"> & Partial<Pick<ChunkVectorEntity<Metadata, Vector>, "chunk_id">>;
+/**
+ * Type for the primary key of chunk vectors
+ */
+export type ChunkVectorKey = {
+    chunk_id: string;
+};
+export type ChunkVectorStorage = IVectorStorage<ChunkRecord, typeof ChunkVectorStorageSchema, ChunkVectorEntity, ChunkVectorPrimaryKey>;
+/**
+ * Search result with score
+ */
+export type ChunkSearchResult = ChunkVectorEntity & {
+    score: number;
+};
+//# sourceMappingURL=ChunkVectorStorageSchema.d.ts.map

package/dist/chunk/ChunkVectorStorageSchema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChunkVectorStorageSchema.d.ts","sourceRoot":"","sources":["../../src/chunk/ChunkVectorStorageSchema.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAA+C,KAAK,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC9F,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD;;;GAGG;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;CAUI,CAAC;AAC1C,MAAM,MAAM,wBAAwB,GAAG,OAAO,wBAAwB,CAAC;AAEvE,eAAO,MAAM,qBAAqB,uBAAwB,CAAC;AAC3D,MAAM,MAAM,qBAAqB,GAAG,OAAO,qBAAqB,CAAC;AAEjE,MAAM,WAAW,iBAAiB,CAChC,QAAQ,SAAS,WAAW,GAAG,WAAW,EAC1C,MAAM,SAAS,UAAU,GAAG,UAAU;IAEtC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,MAAM,uBAAuB,CACjC,QAAQ,SAAS,WAAW,GAAG,WAAW,EAC1C,MAAM,SAAS,UAAU,GAAG,UAAU,IACpC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,UAAU,CAAC,GACvD,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;AAEjE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAElD,MAAM,MAAM,kBAAkB,GAAG,cAAc,CAC7C,WAAW,EACX,OAAO,wBAAwB,EAC/B,iBAAiB,EACjB,qBAAqB,CACtB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,iBAAiB,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC"}

package/dist/common.d.ts

@@ -3,15 +3,15 @@
  * Copyright 2025 Steven Roussey <sroussey@gmail.com>
  * SPDX-License-Identifier: Apache-2.0
  */
+export * from "./chunk/ChunkSchema";
+export * from "./chunk/ChunkVectorStorageSchema";
+export * from "./knowledge-base/KnowledgeBase";
+export * from "./knowledge-base/KnowledgeBaseRegistry";
+export * from "./knowledge-base/createKnowledgeBase";
 export * from "./util/DatasetSchema";
 export * from "./document/Document";
-export * from "./document/DocumentDataset";
-export * from "./document/DocumentDatasetRegistry";
 export * from "./document/DocumentNode";
 export * from "./document/DocumentSchema";
 export * from "./document/DocumentStorageSchema";
 export * from "./document/StructuralParser";
-export * from "./document-chunk/DocumentChunkDataset";
-export * from "./document-chunk/DocumentChunkDatasetRegistry";
-export * from "./document-chunk/DocumentChunkSchema";
 //# sourceMappingURL=common.d.ts.map

package/dist/common.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../src/common.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,sBAAsB,CAAC;AAErC,cAAc,qBAAqB,CAAC;AACpC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,oCAAoC,CAAC;AACnD,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,kCAAkC,CAAC;AACjD,cAAc,6BAA6B,CAAC;AAE5C,cAAc,uCAAuC,CAAC;AACtD,cAAc,+CAA+C,CAAC;AAC9D,cAAc,sCAAsC,CAAC"}
+{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../src/common.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,qBAAqB,CAAC;AACpC,cAAc,kCAAkC,CAAC;AACjD,cAAc,gCAAgC,CAAC;AAC/C,cAAc,wCAAwC,CAAC;AACvD,cAAc,sCAAsC,CAAC;AAGrD,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,kCAAkC,CAAC;AACjD,cAAc,6BAA6B,CAAC"}

package/dist/document/Document.d.ts

@@ -3,7 +3,8 @@
  * Copyright 2025 Steven Roussey <sroussey@gmail.com>
  * SPDX-License-Identifier: Apache-2.0
  */
-import type { ChunkNode, DocumentMetadata, DocumentNode } from "./DocumentSchema";
+import type { ChunkRecord } from "../chunk/ChunkSchema";
+import type { DocumentMetadata, DocumentNode } from "./DocumentSchema";
 /**
  * Document represents a hierarchical document with chunks
  *
@@ -17,15 +18,15 @@ export declare class Document {
     readonly metadata: DocumentMetadata;
     readonly root: DocumentNode;
     private chunks;
-    constructor(root: DocumentNode, metadata: DocumentMetadata, chunks?: ChunkNode[], doc_id?: string);
+    constructor(root: DocumentNode, metadata: DocumentMetadata, chunks?: ChunkRecord[], doc_id?: string);
     /**
      * Set chunks for the document
      */
-    setChunks(chunks: ChunkNode[]): void;
+    setChunks(chunks: ChunkRecord[]): void;
     /**
      * Get all chunks
      */
-    getChunks(): ChunkNode[];
+    getChunks(): ChunkRecord[];
     /**
      * Set the document ID
      */
@@ -33,14 +34,14 @@ export declare class Document {
     /**
      * Find chunks by nodeId
      */
-    findChunksByNodeId(nodeId: string): ChunkNode[];
+    findChunksByNodeId(nodeId: string): ChunkRecord[];
     /**
      * Serialize to JSON
      */
     toJSON(): {
         metadata: DocumentMetadata;
         root: DocumentNode;
-        chunks: ChunkNode[];
+        chunks: ChunkRecord[];
     };
     /**
      * Deserialize from JSON

package/dist/document/Document.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Document.d.ts","sourceRoot":"","sources":["../../src/document/Document.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAElF;;;;;;;GAOG;AACH,qBAAa,QAAQ;IACZ,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,SAAgB,QAAQ,EAAE,gBAAgB,CAAC;IAC3C,SAAgB,IAAI,EAAE,YAAY,CAAC;IACnC,OAAO,CAAC,MAAM,CAAc;gBAG1B,IAAI,EAAE,YAAY,EAClB,QAAQ,EAAE,gBAAgB,EAC1B,MAAM,GAAE,SAAS,EAAO,EACxB,MAAM,CAAC,EAAE,MAAM;IAQjB;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI;IAIpC;;OAEG;IACH,SAAS,IAAI,SAAS,EAAE;IAIxB;;OAEG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAI9B;;OAEG;IACH,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE;IAI/C;;OAEG;IACH,MAAM,IAAI;QACR,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,IAAI,EAAE,YAAY,CAAC;QACnB,MAAM,EAAE,SAAS,EAAE,CAAC;KACrB;IAQD;;OAEG;IACH,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,QAAQ;CAIzD"}
+{"version":3,"file":"Document.d.ts","sourceRoot":"","sources":["../../src/document/Document.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEvE;;;;;;;GAOG;AACH,qBAAa,QAAQ;IACZ,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,SAAgB,QAAQ,EAAE,gBAAgB,CAAC;IAC3C,SAAgB,IAAI,EAAE,YAAY,CAAC;IACnC,OAAO,CAAC,MAAM,CAAgB;gBAG5B,IAAI,EAAE,YAAY,EAClB,QAAQ,EAAE,gBAAgB,EAC1B,MAAM,GAAE,WAAW,EAAO,EAC1B,MAAM,CAAC,EAAE,MAAM;IAQjB;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI;IAItC;;OAEG;IACH,SAAS,IAAI,WAAW,EAAE;IAI1B;;OAEG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAI9B;;OAEG;IACH,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE;IAIjD;;OAEG;IACH,MAAM,IAAI;QACR,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,IAAI,EAAE,YAAY,CAAC;QACnB,MAAM,EAAE,WAAW,EAAE,CAAC;KACvB;IAQD;;OAEG;IACH,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,QAAQ;CAIzD"}