@workglow/ai 0.0.85 → 0.0.87

package/dist/task/HierarchicalChunkerTask.d.ts

@@ -0,0 +1,212 @@
+/**
+ * @license
+ * Copyright 2025 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { CreateWorkflow, IExecuteContext, JobQueueTaskConfig, Task } from "@workglow/task-graph";
+import { DataPortSchema, FromSchema } from "@workglow/util";
+declare const inputSchema: {
+    readonly type: "object";
+    readonly properties: {
+        readonly doc_id: {
+            readonly type: "string";
+            readonly title: "Document ID";
+            readonly description: "The ID of the document";
+        };
+        readonly documentTree: {
+            readonly title: "Document Tree";
+            readonly description: "The hierarchical document tree to chunk";
+        };
+        readonly maxTokens: {
+            readonly type: "number";
+            readonly title: "Max Tokens";
+            readonly description: "Maximum tokens per chunk";
+            readonly minimum: 50;
+            readonly default: 512;
+        };
+        readonly overlap: {
+            readonly type: "number";
+            readonly title: "Overlap";
+            readonly description: "Overlap in tokens between chunks";
+            readonly minimum: 0;
+            readonly default: 50;
+        };
+        readonly reservedTokens: {
+            readonly type: "number";
+            readonly title: "Reserved Tokens";
+            readonly description: "Reserved tokens for metadata/wrappers";
+            readonly minimum: 0;
+            readonly default: 10;
+        };
+        readonly strategy: {
+            readonly type: "string";
+            readonly enum: readonly ["hierarchical", "flat", "sentence"];
+            readonly title: "Chunking Strategy";
+            readonly description: "Strategy for chunking";
+            readonly default: "hierarchical";
+        };
+    };
+    readonly required: readonly [];
+    readonly additionalProperties: false;
+};
+declare const outputSchema: {
+    readonly type: "object";
+    readonly properties: {
+        readonly doc_id: {
+            readonly type: "string";
+            readonly title: "Document ID";
+            readonly description: "The document ID (passed through)";
+        };
+        readonly chunks: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "object";
+                readonly properties: {
+                    readonly chunkId: {
+                        readonly type: "string";
+                        readonly title: "Chunk ID";
+                        readonly description: "Unique identifier for this chunk";
+                    };
+                    readonly doc_id: {
+                        readonly type: "string";
+                        readonly title: "Document ID";
+                        readonly description: "ID of the parent document";
+                    };
+                    readonly text: {
+                        readonly type: "string";
+                        readonly title: "Text";
+                        readonly description: "Text content of the chunk";
+                    };
+                    readonly nodePath: {
+                        readonly type: "array";
+                        readonly items: {
+                            readonly type: "string";
+                        };
+                        readonly title: "Node Path";
+                        readonly description: "Node IDs from root to leaf";
+                    };
+                    readonly depth: {
+                        readonly type: "integer";
+                        readonly title: "Depth";
+                        readonly description: "Depth in the document tree";
+                    };
+                    readonly enrichment: {
+                        readonly type: "object";
+                        readonly properties: {
+                            readonly summary: {
+                                readonly type: "string";
+                                readonly title: "Summary";
+                                readonly description: "Summary of the chunk content";
+                            };
+                            readonly entities: {
+                                readonly type: "array";
+                                readonly items: {
+                                    readonly type: "object";
+                                    readonly properties: {
+                                        readonly text: {
+                                            readonly type: "string";
+                                            readonly title: "Text";
+                                            readonly description: "Entity text";
+                                        };
+                                        readonly type: {
+                                            readonly type: "string";
+                                            readonly title: "Type";
+                                            readonly description: "Entity type (e.g., PERSON, ORG, LOC)";
+                                        };
+                                        readonly score: {
+                                            readonly type: "number";
+                                            readonly title: "Score";
+                                            readonly description: "Confidence score";
+                                        };
+                                    };
+                                    readonly required: readonly ["text", "type", "score"];
+                                    readonly additionalProperties: false;
+                                };
+                                readonly title: "Entities";
+                                readonly description: "Named entities extracted from the chunk";
+                            };
+                        };
+                        readonly additionalProperties: false;
+                    };
+                };
+                readonly required: readonly ["chunkId", "doc_id", "text", "nodePath", "depth"];
+                readonly additionalProperties: false;
+            };
+            readonly title: "Chunks";
+            readonly description: "Array of chunk nodes";
+        };
+        readonly text: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "string";
+            };
+            readonly title: "Texts";
+            readonly description: "Chunk texts (for TextEmbeddingTask)";
+        };
+        readonly count: {
+            readonly type: "number";
+            readonly title: "Count";
+            readonly description: "Number of chunks generated";
+        };
+    };
+    readonly required: readonly ["doc_id", "chunks", "text", "count"];
+    readonly additionalProperties: false;
+};
+export type HierarchicalChunkerTaskInput = FromSchema<typeof inputSchema>;
+export type HierarchicalChunkerTaskOutput = FromSchema<typeof outputSchema>;
+/**
+ * Task for hierarchical chunking that respects token budgets and document structure
+ */
+export declare class HierarchicalChunkerTask extends Task<HierarchicalChunkerTaskInput, HierarchicalChunkerTaskOutput, JobQueueTaskConfig> {
+    static type: string;
+    static category: string;
+    static title: string;
+    static description: string;
+    static cacheable: boolean;
+    static inputSchema(): DataPortSchema;
+    static outputSchema(): DataPortSchema;
+    execute(input: HierarchicalChunkerTaskInput, context: IExecuteContext): Promise<HierarchicalChunkerTaskOutput>;
+    /**
+     * Hierarchical chunking that respects document structure
+     */
+    private chunkHierarchically;
+    /**
+     * Chunk a single text string
+     */
+    private chunkText;
+    /**
+     * Flat chunking (ignores hierarchy)
+     */
+    private chunkFlat;
+    /**
+     * Collect all text from a node and its descendants
+     */
+    private collectAllText;
+}
+export declare const hierarchicalChunker: (input: HierarchicalChunkerTaskInput, config?: JobQueueTaskConfig) => Promise<{
+    doc_id: string;
+    chunks: {
+        enrichment?: {
+            summary?: string | undefined;
+            entities?: {
+                type: string;
+                text: string;
+                score: number;
+            }[] | undefined;
+        } | undefined;
+        doc_id: string;
+        chunkId: string;
+        text: string;
+        nodePath: string[];
+        depth: number;
+    }[];
+    text: string[];
+    count: number;
+}>;
+declare module "@workglow/task-graph" {
+    interface Workflow {
+        hierarchicalChunker: CreateWorkflow<HierarchicalChunkerTaskInput, HierarchicalChunkerTaskOutput, JobQueueTaskConfig>;
+    }
+}
+export {};
+//# sourceMappingURL=HierarchicalChunkerTask.d.ts.map

package/dist/task/HierarchicalChunkerTask.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"HierarchicalChunkerTask.d.ts","sourceRoot":"","sources":["../../src/task/HierarchicalChunkerTask.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAWH,OAAO,EACL,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,IAAI,EAEL,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,cAAc,EAAE,UAAU,EAAS,MAAM,gBAAgB,CAAC;AAEnE,QAAA,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2CkB,CAAC;AAEpC,QAAA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4BiB,CAAC;AAEpC,MAAM,MAAM,4BAA4B,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC;AAC1E,MAAM,MAAM,6BAA6B,GAAG,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC;AAE5E;;GAEG;AACH,qBAAa,uBAAwB,SAAQ,IAAI,CAC/C,4BAA4B,EAC5B,6BAA6B,EAC7B,kBAAkB,CACnB;IACC,OAAc,IAAI,SAA6B;IAC/C,OAAc,QAAQ,SAAc;IACpC,OAAc,KAAK,SAA0B;IAC7C,OAAc,WAAW,SAA6D;IACtF,OAAc,SAAS,UAAQ;WAEjB,WAAW,IAAI,cAAc;WAI7B,YAAY,IAAI,cAAc;IAItC,OAAO,CACX,KAAK,EAAE,4BAA4B,EACnC,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,6BAA6B,CAAC;IAwCzC;;OAEG;YACW,mBAAmB;IAsBjC;;OAEG;YACW,SAAS;IAoDvB;;OAEG;YACW,SAAS;IAWvB;;OAEG;IACH,OAAO,CAAC,cAAc;CAgBvB;AAED,eAAO,MAAM,mBAAmB,GAC9B,OAAO,4BAA4B,EACnC,SAAS,kBAAkB;;;;;;;;;;;;;;;;;;;EAG5B,CAAC;AAEF,OAAO,QAAQ,sBAAsB,CAAC;IACpC,UAAU,QAAQ;QAChB,mBAAmB,EAAE,cAAc,CACjC,4BAA4B,EAC5B,6BAA6B,EAC7B,kBAAkB,CACnB,CAAC;KACH;CACF"}

package/dist/task/HierarchyJoinTask.d.ts

@@ -0,0 +1,318 @@
+/**
+ * @license
+ * Copyright 2025 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { CreateWorkflow, IExecuteContext, JobQueueTaskConfig, Task } from "@workglow/task-graph";
+import { DataPortSchema, FromSchema } from "@workglow/util";
+declare const inputSchema: {
+    readonly type: "object";
+    readonly properties: {
+        readonly documents: {
+            readonly title: "Document Dataset";
+            readonly description: "Dataset ID or instance for document data storage";
+        } & {
+            title: string;
+            description: string;
+        } & {
+            readonly format: "dataset:document";
+            readonly anyOf: readonly [{
+                readonly type: "string";
+                readonly title: "Dataset ID";
+            }, {
+                readonly title: "Dataset Instance";
+                readonly additionalProperties: true;
+            }];
+        };
+        readonly chunks: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "string";
+            };
+            readonly title: "Chunks";
+            readonly description: "Retrieved text chunks";
+        };
+        readonly chunk_ids: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "string";
+            };
+            readonly title: "Chunk IDs";
+            readonly description: "IDs of retrieved chunks";
+        };
+        readonly metadata: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "object";
+                readonly properties: {
+                    readonly doc_id: {
+                        readonly type: "string";
+                        readonly title: "Document ID";
+                        readonly description: "ID of the parent document";
+                    };
+                    readonly chunkId: {
+                        readonly type: "string";
+                        readonly title: "Chunk ID";
+                        readonly description: "Unique identifier for this chunk";
+                    };
+                    readonly leafNodeId: {
+                        readonly type: "string";
+                        readonly title: "Leaf Node ID";
+                        readonly description: "ID of the leaf node this chunk belongs to";
+                    };
+                    readonly depth: {
+                        readonly type: "integer";
+                        readonly title: "Depth";
+                        readonly description: "Depth in the document tree";
+                    };
+                    readonly text: {
+                        readonly type: "string";
+                        readonly title: "Text";
+                        readonly description: "Text content of the chunk";
+                    };
+                    readonly nodePath: {
+                        readonly type: "array";
+                        readonly items: {
+                            readonly type: "string";
+                        };
+                        readonly title: "Node Path";
+                        readonly description: "Node IDs from root to leaf";
+                    };
+                    readonly summary: {
+                        readonly type: "string";
+                        readonly title: "Summary";
+                        readonly description: "Summary of the chunk content";
+                    };
+                    readonly entities: {
+                        readonly type: "array";
+                        readonly items: {
+                            readonly type: "object";
+                            readonly properties: {
+                                readonly text: {
+                                    readonly type: "string";
+                                    readonly title: "Text";
+                                    readonly description: "Entity text";
+                                };
+                                readonly type: {
+                                    readonly type: "string";
+                                    readonly title: "Type";
+                                    readonly description: "Entity type (e.g., PERSON, ORG, LOC)";
+                                };
+                                readonly score: {
+                                    readonly type: "number";
+                                    readonly title: "Score";
+                                    readonly description: "Confidence score";
+                                };
+                            };
+                            readonly required: readonly ["text", "type", "score"];
+                            readonly additionalProperties: false;
+                        };
+                        readonly title: "Entities";
+                        readonly description: "Named entities extracted from the chunk";
+                    };
+                };
+                readonly required: readonly ["doc_id", "chunkId", "leafNodeId", "depth", "text", "nodePath"];
+                readonly additionalProperties: true;
+            };
+            readonly title: "Chunk Metadata";
+            readonly description: "Metadata for each chunk";
+        };
+        readonly scores: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "number";
+            };
+            readonly title: "Scores";
+            readonly description: "Similarity scores for each result";
+        };
+        readonly includeParentSummaries: {
+            readonly type: "boolean";
+            readonly title: "Include Parent Summaries";
+            readonly description: "Whether to include summaries from parent nodes";
+            readonly default: true;
+        };
+        readonly includeEntities: {
+            readonly type: "boolean";
+            readonly title: "Include Entities";
+            readonly description: "Whether to include entities from the node hierarchy";
+            readonly default: true;
+        };
+    };
+    readonly required: readonly ["documents", "chunks", "chunk_ids", "metadata", "scores"];
+    readonly additionalProperties: false;
+};
+declare const outputSchema: {
+    readonly type: "object";
+    readonly properties: {
+        readonly chunks: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "string";
+            };
+            readonly title: "Chunks";
+            readonly description: "Retrieved text chunks";
+        };
+        readonly chunk_ids: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "string";
+            };
+            readonly title: "Chunk IDs";
+            readonly description: "IDs of retrieved chunks";
+        };
+        readonly metadata: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "object";
+                readonly properties: {
+                    readonly doc_id: {
+                        readonly type: "string";
+                        readonly title: "Document ID";
+                        readonly description: "ID of the parent document";
+                    };
+                    readonly chunkId: {
+                        readonly type: "string";
+                        readonly title: "Chunk ID";
+                        readonly description: "Unique identifier for this chunk";
+                    };
+                    readonly leafNodeId: {
+                        readonly type: "string";
+                        readonly title: "Leaf Node ID";
+                        readonly description: "ID of the leaf node this chunk belongs to";
+                    };
+                    readonly depth: {
+                        readonly type: "integer";
+                        readonly title: "Depth";
+                        readonly description: "Depth in the document tree";
+                    };
+                    readonly text: {
+                        readonly type: "string";
+                        readonly title: "Text";
+                        readonly description: "Text content of the chunk";
+                    };
+                    readonly nodePath: {
+                        readonly type: "array";
+                        readonly items: {
+                            readonly type: "string";
+                        };
+                        readonly title: "Node Path";
+                        readonly description: "Node IDs from root to leaf";
+                    };
+                    readonly summary: {
+                        readonly type: "string";
+                        readonly title: "Summary";
+                        readonly description: "Summary of the chunk content";
+                    };
+                    readonly entities: {
+                        readonly type: "array";
+                        readonly items: {
+                            readonly type: "object";
+                            readonly properties: {
+                                readonly text: {
+                                    readonly type: "string";
+                                    readonly title: "Text";
+                                    readonly description: "Entity text";
+                                };
+                                readonly type: {
+                                    readonly type: "string";
+                                    readonly title: "Type";
+                                    readonly description: "Entity type (e.g., PERSON, ORG, LOC)";
+                                };
+                                readonly score: {
+                                    readonly type: "number";
+                                    readonly title: "Score";
+                                    readonly description: "Confidence score";
+                                };
+                            };
+                            readonly required: readonly ["text", "type", "score"];
+                            readonly additionalProperties: false;
+                        };
+                        readonly title: "Entities";
+                        readonly description: "Named entities (rolled up from hierarchy)";
+                    };
+                    readonly parentSummaries: {
+                        readonly type: "array";
+                        readonly items: {
+                            readonly type: "string";
+                        };
+                        readonly title: "Parent Summaries";
+                        readonly description: "Summaries from ancestor nodes";
+                    };
+                    readonly sectionTitles: {
+                        readonly type: "array";
+                        readonly items: {
+                            readonly type: "string";
+                        };
+                        readonly title: "Section Titles";
+                        readonly description: "Titles of ancestor section nodes";
+                    };
+                };
+                readonly required: readonly ["doc_id", "chunkId", "leafNodeId", "depth", "text", "nodePath"];
+                readonly additionalProperties: true;
+            };
+            readonly title: "Enriched Metadata";
+            readonly description: "Metadata enriched with hierarchy information";
+        };
+        readonly scores: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "number";
+            };
+            readonly title: "Scores";
+            readonly description: "Similarity scores";
+        };
+        readonly count: {
+            readonly type: "number";
+            readonly title: "Count";
+            readonly description: "Number of results";
+        };
+    };
+    readonly required: readonly ["chunks", "chunk_ids", "metadata", "scores", "count"];
+    readonly additionalProperties: false;
+};
+export type HierarchyJoinTaskInput = FromSchema<typeof inputSchema>;
+export type HierarchyJoinTaskOutput = FromSchema<typeof outputSchema>;
+/**
+ * Task for enriching search results with hierarchy information
+ * Joins chunk IDs back to document repository to get parent summaries and entities
+ */
+export declare class HierarchyJoinTask extends Task<HierarchyJoinTaskInput, HierarchyJoinTaskOutput, JobQueueTaskConfig> {
+    static type: string;
+    static category: string;
+    static title: string;
+    static description: string;
+    static cacheable: boolean;
+    static inputSchema(): DataPortSchema;
+    static outputSchema(): DataPortSchema;
+    execute(input: HierarchyJoinTaskInput, context: IExecuteContext): Promise<HierarchyJoinTaskOutput>;
+}
+export declare const hierarchyJoin: (input: HierarchyJoinTaskInput, config?: JobQueueTaskConfig) => Promise<{
+    metadata: {
+        [x: string]: unknown;
+        summary?: string | undefined;
+        entities?: {
+            type: string;
+            text: string;
+            score: number;
+        }[] | undefined;
+        parentSummaries?: string[] | undefined;
+        sectionTitles?: string[] | undefined;
+        doc_id: string;
+        chunkId: string;
+        text: string;
+        nodePath: string[];
+        depth: number;
+        leafNodeId: string;
+    }[];
+    chunks: string[];
+    scores: number[];
+    chunk_ids: string[];
+    count: number;
+}>;
+declare module "@workglow/task-graph" {
+    interface Workflow {
+        hierarchyJoin: CreateWorkflow<HierarchyJoinTaskInput, HierarchyJoinTaskOutput, JobQueueTaskConfig>;
+    }
+}
+export {};
+//# sourceMappingURL=HierarchyJoinTask.d.ts.map

package/dist/task/HierarchyJoinTask.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"HierarchyJoinTask.d.ts","sourceRoot":"","sources":["../../src/task/HierarchyJoinTask.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AASH,OAAO,EACL,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,IAAI,EAEL,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5D,QAAA,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyCkB,CAAC;AAEpC,QAAA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8BiB,CAAC;AAEpC,MAAM,MAAM,sBAAsB,GAAG,UAAU,CAAC,OAAO,WAAW,CAAC,CAAC;AACpE,MAAM,MAAM,uBAAuB,GAAG,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC;AAEtE;;;GAGG;AACH,qBAAa,iBAAkB,SAAQ,IAAI,CACzC,sBAAsB,EACtB,uBAAuB,EACvB,kBAAkB,CACnB;IACC,OAAc,IAAI,SAAuB;IACzC,OAAc,QAAQ,SAAS;IAC/B,OAAc,KAAK,SAAoB;IACvC,OAAc,WAAW,SAA2D;IACpF,OAAc,SAAS,UAAS;WAElB,WAAW,IAAI,cAAc;WAI7B,YAAY,IAAI,cAAc;IAItC,OAAO,CACX,KAAK,EAAE,sBAAsB,EAC7B,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,uBAAuB,CAAC;CAsGpC;AAED,eAAO,MAAM,aAAa,GAAI,OAAO,sBAAsB,EAAE,SAAS,kBAAkB;;;;;;;;;;;;;;;;;;;;;;EAEvF,CAAC;AAEF,OAAO,QAAQ,sBAAsB,CAAC;IACpC,UAAU,QAAQ;QAChB,aAAa,EAAE,cAAc,CAC3B,sBAAsB,EACtB,uBAAuB,EACvB,kBAAkB,CACnB,CAAC;KACH;CACF"}