@workglow/ai 0.2.15 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -15
- package/dist/browser.js +544 -1033
- package/dist/browser.js.map +17 -20
- package/dist/bun.js +544 -1033
- package/dist/bun.js.map +17 -20
- package/dist/node.js +544 -1033
- package/dist/node.js.map +17 -20
- package/dist/provider/AiProvider.d.ts +8 -8
- package/dist/provider/AiProvider.d.ts.map +1 -1
- package/dist/provider/AiProviderRegistry.d.ts +16 -16
- package/dist/provider/AiProviderRegistry.d.ts.map +1 -1
- package/dist/task/ChunkRetrievalTask.d.ts +32 -49
- package/dist/task/ChunkRetrievalTask.d.ts.map +1 -1
- package/dist/task/ChunkVectorUpsertTask.d.ts +107 -24
- package/dist/task/ChunkVectorUpsertTask.d.ts.map +1 -1
- package/dist/task/ContextBuilderTask.d.ts +3 -2
- package/dist/task/ContextBuilderTask.d.ts.map +1 -1
- package/dist/task/HierarchyJoinTask.d.ts +44 -42
- package/dist/task/HierarchyJoinTask.d.ts.map +1 -1
- package/dist/task/QueryExpanderTask.d.ts +5 -31
- package/dist/task/QueryExpanderTask.d.ts.map +1 -1
- package/dist/task/RerankerTask.d.ts +7 -89
- package/dist/task/RerankerTask.d.ts.map +1 -1
- package/dist/task/TextChunkerTask.d.ts +139 -37
- package/dist/task/TextChunkerTask.d.ts.map +1 -1
- package/dist/task/VectorQuantizeTask.d.ts +2 -1
- package/dist/task/VectorQuantizeTask.d.ts.map +1 -1
- package/dist/task/VectorSimilarityTask.d.ts +2 -4
- package/dist/task/VectorSimilarityTask.d.ts.map +1 -1
- package/dist/task/base/AiTask.d.ts +4 -4
- package/dist/task/base/AiTask.d.ts.map +1 -1
- package/dist/task/index.d.ts +1 -7
- package/dist/task/index.d.ts.map +1 -1
- package/dist/worker.js +25 -26
- package/dist/worker.js.map +4 -4
- package/package.json +11 -11
- package/dist/task/ChunkToVectorTask.d.ts +0 -210
- package/dist/task/ChunkToVectorTask.d.ts.map +0 -1
- package/dist/task/ChunkVectorHybridSearchTask.d.ts +0 -167
- package/dist/task/ChunkVectorHybridSearchTask.d.ts.map +0 -1
- package/dist/task/ChunkVectorSearchTask.d.ts +0 -139
- package/dist/task/ChunkVectorSearchTask.d.ts.map +0 -1
package/dist/bun.js
CHANGED
|
@@ -15,7 +15,7 @@ import { globalServiceRegistry, WORKER_MANAGER } from "@workglow/util/worker";
|
|
|
15
15
|
class AiProviderRegistry {
|
|
16
16
|
runFnRegistry = new Map;
|
|
17
17
|
streamFnRegistry = new Map;
|
|
18
|
-
|
|
18
|
+
previewRunFnRegistry = new Map;
|
|
19
19
|
providers = new Map;
|
|
20
20
|
strategyResolvers = new Map;
|
|
21
21
|
defaultStrategy;
|
|
@@ -31,7 +31,7 @@ class AiProviderRegistry {
|
|
|
31
31
|
for (const [, providerMap] of this.streamFnRegistry) {
|
|
32
32
|
providerMap.delete(name);
|
|
33
33
|
}
|
|
34
|
-
for (const [, providerMap] of this.
|
|
34
|
+
for (const [, providerMap] of this.previewRunFnRegistry) {
|
|
35
35
|
providerMap.delete(name);
|
|
36
36
|
}
|
|
37
37
|
}
|
|
@@ -109,25 +109,24 @@ class AiProviderRegistry {
|
|
|
109
109
|
const taskTypeMap = this.streamFnRegistry.get(taskType);
|
|
110
110
|
return taskTypeMap?.get(modelProvider);
|
|
111
111
|
}
|
|
112
|
-
|
|
113
|
-
const
|
|
112
|
+
registerAsWorkerPreviewRunFn(modelProvider, taskType) {
|
|
113
|
+
const previewFn = async (input, model) => {
|
|
114
114
|
const workerManager = globalServiceRegistry.get(WORKER_MANAGER);
|
|
115
|
-
return workerManager.
|
|
115
|
+
return workerManager.callWorkerPreviewFunction(modelProvider, taskType, [
|
|
116
116
|
input,
|
|
117
|
-
output,
|
|
118
117
|
model
|
|
119
118
|
]);
|
|
120
119
|
};
|
|
121
|
-
this.
|
|
120
|
+
this.registerPreviewRunFn(modelProvider, taskType, previewFn);
|
|
122
121
|
}
|
|
123
|
-
|
|
124
|
-
if (!this.
|
|
125
|
-
this.
|
|
122
|
+
registerPreviewRunFn(modelProvider, taskType, previewRunFn) {
|
|
123
|
+
if (!this.previewRunFnRegistry.has(taskType)) {
|
|
124
|
+
this.previewRunFnRegistry.set(taskType, new Map);
|
|
126
125
|
}
|
|
127
|
-
this.
|
|
126
|
+
this.previewRunFnRegistry.get(taskType).set(modelProvider, previewRunFn);
|
|
128
127
|
}
|
|
129
|
-
|
|
130
|
-
const taskTypeMap = this.
|
|
128
|
+
getPreviewRunFn(modelProvider, taskType) {
|
|
129
|
+
const taskTypeMap = this.previewRunFnRegistry.get(taskType);
|
|
131
130
|
return taskTypeMap?.get(modelProvider);
|
|
132
131
|
}
|
|
133
132
|
getDirectRunFn(modelProvider, taskType) {
|
|
@@ -722,11 +721,11 @@ function resolveAiProviderGpuQueueConcurrency(concurrency) {
|
|
|
722
721
|
class AiProvider {
|
|
723
722
|
tasks;
|
|
724
723
|
streamTasks;
|
|
725
|
-
|
|
726
|
-
constructor(tasks, streamTasks,
|
|
724
|
+
previewTasks;
|
|
725
|
+
constructor(tasks, streamTasks, previewTasks) {
|
|
727
726
|
this.tasks = tasks;
|
|
728
727
|
this.streamTasks = streamTasks;
|
|
729
|
-
this.
|
|
728
|
+
this.previewTasks = previewTasks;
|
|
730
729
|
}
|
|
731
730
|
get supportedTaskTypes() {
|
|
732
731
|
return this.taskTypes;
|
|
@@ -737,8 +736,8 @@ class AiProvider {
|
|
|
737
736
|
getStreamFn(taskType) {
|
|
738
737
|
return this.streamTasks?.[taskType];
|
|
739
738
|
}
|
|
740
|
-
|
|
741
|
-
return this.
|
|
739
|
+
getPreviewRunFn(taskType) {
|
|
740
|
+
return this.previewTasks?.[taskType];
|
|
742
741
|
}
|
|
743
742
|
async register(options = {}) {
|
|
744
743
|
const isInline = !!this.tasks;
|
|
@@ -766,7 +765,7 @@ class AiProvider {
|
|
|
766
765
|
for (const taskType of this.taskTypes) {
|
|
767
766
|
registry.registerAsWorkerRunFn(this.name, taskType);
|
|
768
767
|
registry.registerAsWorkerStreamFn(this.name, taskType);
|
|
769
|
-
registry.
|
|
768
|
+
registry.registerAsWorkerPreviewRunFn(this.name, taskType);
|
|
770
769
|
}
|
|
771
770
|
} else {
|
|
772
771
|
for (const [taskType, fn] of Object.entries(this.tasks)) {
|
|
@@ -778,9 +777,9 @@ class AiProvider {
|
|
|
778
777
|
}
|
|
779
778
|
}
|
|
780
779
|
}
|
|
781
|
-
if (this.
|
|
782
|
-
for (const [taskType, fn] of Object.entries(this.
|
|
783
|
-
registry.
|
|
780
|
+
if (this.previewTasks) {
|
|
781
|
+
for (const [taskType, fn] of Object.entries(this.previewTasks)) {
|
|
782
|
+
registry.registerPreviewRunFn(this.name, taskType, fn);
|
|
784
783
|
}
|
|
785
784
|
}
|
|
786
785
|
registry.registerProvider(this);
|
|
@@ -803,9 +802,9 @@ class AiProvider {
|
|
|
803
802
|
workerServer.registerStreamFunction(taskType, fn);
|
|
804
803
|
}
|
|
805
804
|
}
|
|
806
|
-
if (this.
|
|
807
|
-
for (const [taskType, fn] of Object.entries(this.
|
|
808
|
-
workerServer.
|
|
805
|
+
if (this.previewTasks) {
|
|
806
|
+
for (const [taskType, fn] of Object.entries(this.previewTasks)) {
|
|
807
|
+
workerServer.registerPreviewFunction(taskType, fn);
|
|
809
808
|
}
|
|
810
809
|
}
|
|
811
810
|
}
|
|
@@ -1112,16 +1111,16 @@ class AiTask extends Task {
|
|
|
1112
1111
|
const model = input.model;
|
|
1113
1112
|
return model?.provider;
|
|
1114
1113
|
}
|
|
1115
|
-
async
|
|
1114
|
+
async executePreview(input, context) {
|
|
1116
1115
|
const model = input.model;
|
|
1117
1116
|
if (model && typeof model === "object" && model.provider) {
|
|
1118
1117
|
const taskType = this.constructor.runtype ?? this.constructor.type;
|
|
1119
|
-
const
|
|
1120
|
-
if (
|
|
1121
|
-
return
|
|
1118
|
+
const previewFn = getAiProviderRegistry().getPreviewRunFn(model.provider, taskType);
|
|
1119
|
+
if (previewFn) {
|
|
1120
|
+
return previewFn(input, model);
|
|
1122
1121
|
}
|
|
1123
1122
|
}
|
|
1124
|
-
return super.
|
|
1123
|
+
return super.executePreview(input, context);
|
|
1125
1124
|
}
|
|
1126
1125
|
async validateInput(input) {
|
|
1127
1126
|
const inputSchema = this.inputSchema();
|
|
@@ -1708,7 +1707,7 @@ var inputSchema = {
|
|
|
1708
1707
|
title: "Knowledge Base",
|
|
1709
1708
|
description: "The knowledge base instance to search in"
|
|
1710
1709
|
}),
|
|
1711
|
-
query:
|
|
1710
|
+
query: {
|
|
1712
1711
|
oneOf: [
|
|
1713
1712
|
{ type: "string" },
|
|
1714
1713
|
TypedArraySchema2({
|
|
@@ -1717,12 +1716,19 @@ var inputSchema = {
|
|
|
1717
1716
|
})
|
|
1718
1717
|
],
|
|
1719
1718
|
title: "Query",
|
|
1720
|
-
description: "Query string or pre-computed query vector"
|
|
1721
|
-
}
|
|
1719
|
+
description: "Query string (requires `model`) or pre-computed query vector"
|
|
1720
|
+
},
|
|
1722
1721
|
model: TypeModel("model:TextEmbeddingTask", {
|
|
1723
1722
|
title: "Model",
|
|
1724
1723
|
description: "Text embedding model to use for query embedding (required when query is a string)"
|
|
1725
1724
|
}),
|
|
1725
|
+
method: {
|
|
1726
|
+
type: "string",
|
|
1727
|
+
enum: ["similarity", "hybrid"],
|
|
1728
|
+
title: "Retrieval Method",
|
|
1729
|
+
description: "Retrieval strategy: 'similarity' (vector only) or 'hybrid' (vector + full-text).",
|
|
1730
|
+
default: "similarity"
|
|
1731
|
+
},
|
|
1726
1732
|
topK: {
|
|
1727
1733
|
type: "number",
|
|
1728
1734
|
title: "Top K",
|
|
@@ -1743,6 +1749,14 @@ var inputSchema = {
|
|
|
1743
1749
|
maximum: 1,
|
|
1744
1750
|
default: 0
|
|
1745
1751
|
},
|
|
1752
|
+
vectorWeight: {
|
|
1753
|
+
type: "number",
|
|
1754
|
+
title: "Vector Weight",
|
|
1755
|
+
description: "For hybrid method: weight for vector similarity (0-1), remainder goes to text relevance",
|
|
1756
|
+
minimum: 0,
|
|
1757
|
+
maximum: 1,
|
|
1758
|
+
default: 0.7
|
|
1759
|
+
},
|
|
1746
1760
|
returnVectors: {
|
|
1747
1761
|
type: "boolean",
|
|
1748
1762
|
title: "Return Vectors",
|
|
@@ -1807,7 +1821,7 @@ var outputSchema = {
|
|
|
1807
1821
|
title: "Count",
|
|
1808
1822
|
description: "Number of results returned"
|
|
1809
1823
|
},
|
|
1810
|
-
query:
|
|
1824
|
+
query: {
|
|
1811
1825
|
oneOf: [
|
|
1812
1826
|
{ type: "string" },
|
|
1813
1827
|
TypedArraySchema2({
|
|
@@ -1817,7 +1831,7 @@ var outputSchema = {
|
|
|
1817
1831
|
],
|
|
1818
1832
|
title: "Query",
|
|
1819
1833
|
description: "The query used for retrieval (pass-through)"
|
|
1820
|
-
}
|
|
1834
|
+
}
|
|
1821
1835
|
},
|
|
1822
1836
|
required: ["chunks", "chunk_ids", "metadata", "scores", "count", "query"],
|
|
1823
1837
|
additionalProperties: false
|
|
@@ -1827,7 +1841,7 @@ class ChunkRetrievalTask extends Task2 {
|
|
|
1827
1841
|
static type = "ChunkRetrievalTask";
|
|
1828
1842
|
static category = "RAG";
|
|
1829
1843
|
static title = "Chunk Retrieval";
|
|
1830
|
-
static description = "End-to-end retrieval: embed query and search
|
|
1844
|
+
static description = "End-to-end retrieval: embed query (if string) and search the knowledge base. Supports similarity and hybrid methods.";
|
|
1831
1845
|
static cacheable = true;
|
|
1832
1846
|
static inputSchema() {
|
|
1833
1847
|
return inputSchema;
|
|
@@ -1842,332 +1856,46 @@ class ChunkRetrievalTask extends Task2 {
|
|
|
1842
1856
|
topK = 5,
|
|
1843
1857
|
filter,
|
|
1844
1858
|
model,
|
|
1859
|
+
method = "similarity",
|
|
1860
|
+
vectorWeight = 0.7,
|
|
1845
1861
|
scoreThreshold = 0,
|
|
1846
1862
|
returnVectors = false
|
|
1847
1863
|
} = input;
|
|
1848
1864
|
const kb = knowledgeBase;
|
|
1849
|
-
|
|
1850
|
-
if (
|
|
1865
|
+
const queryIsString = typeof query === "string";
|
|
1866
|
+
if (method === "hybrid" && !queryIsString) {
|
|
1867
|
+
throw new Error("Hybrid retrieval requires a string query (it will be embedded and used for full-text search).");
|
|
1868
|
+
}
|
|
1869
|
+
if (method === "hybrid" && !kb.supportsHybridSearch()) {
|
|
1870
|
+
throw new Error("The provided knowledge base does not support hybrid search. Use method: 'similarity' or a backend with hybrid support (e.g., Postgres with pgvector).");
|
|
1871
|
+
}
|
|
1872
|
+
let queryVector;
|
|
1873
|
+
let queryText;
|
|
1874
|
+
if (queryIsString) {
|
|
1851
1875
|
if (!model) {
|
|
1852
1876
|
throw new Error("Model is required when query is a string. Please provide a model with format 'model:TextEmbeddingTask'.");
|
|
1853
1877
|
}
|
|
1878
|
+
queryText = query;
|
|
1854
1879
|
const embeddingTask = context.own(new TextEmbeddingTask);
|
|
1855
1880
|
const embeddingResult = await embeddingTask.run({ text: query, model });
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1881
|
+
const vec = embeddingResult.vector;
|
|
1882
|
+
queryVector = Array.isArray(vec) ? vec[0] : vec;
|
|
1883
|
+
} else if (isTypedArray(query)) {
|
|
1884
|
+
queryVector = query;
|
|
1859
1885
|
} else {
|
|
1860
|
-
throw new Error("Query must be a string
|
|
1861
|
-
}
|
|
1862
|
-
const searchVectors = queryVectors.map((v) => v instanceof Float32Array ? v : new Float32Array(v));
|
|
1863
|
-
const results = [];
|
|
1864
|
-
for (const searchVector of searchVectors) {
|
|
1865
|
-
const res = await kb.similaritySearch(searchVector, {
|
|
1866
|
-
topK,
|
|
1867
|
-
filter,
|
|
1868
|
-
scoreThreshold
|
|
1869
|
-
});
|
|
1870
|
-
results.push(...res);
|
|
1871
|
-
}
|
|
1872
|
-
const chunks = results.map((r) => {
|
|
1873
|
-
const meta = r.metadata;
|
|
1874
|
-
return meta.text || JSON.stringify(meta);
|
|
1875
|
-
});
|
|
1876
|
-
const output = {
|
|
1877
|
-
chunks,
|
|
1878
|
-
chunk_ids: results.map((r) => r.chunk_id),
|
|
1879
|
-
metadata: results.map((r) => r.metadata),
|
|
1880
|
-
scores: results.map((r) => r.score),
|
|
1881
|
-
count: results.length,
|
|
1882
|
-
query
|
|
1883
|
-
};
|
|
1884
|
-
if (returnVectors) {
|
|
1885
|
-
output.vectors = results.map((r) => r.vector);
|
|
1886
|
+
throw new Error("Query must be a string or TypedArray");
|
|
1886
1887
|
}
|
|
1887
|
-
return output;
|
|
1888
|
-
}
|
|
1889
|
-
}
|
|
1890
|
-
var chunkRetrieval = (input, config) => {
|
|
1891
|
-
return new ChunkRetrievalTask(config).run(input);
|
|
1892
|
-
};
|
|
1893
|
-
Workflow3.prototype.chunkRetrieval = CreateWorkflow3(ChunkRetrievalTask);
|
|
1894
|
-
|
|
1895
|
-
// src/task/ChunkToVectorTask.ts
|
|
1896
|
-
import { ChunkRecordSchema } from "@workglow/knowledge-base";
|
|
1897
|
-
import { CreateWorkflow as CreateWorkflow4, Task as Task3, Workflow as Workflow4 } from "@workglow/task-graph";
|
|
1898
|
-
import {
|
|
1899
|
-
TypedArraySchema as TypedArraySchema3
|
|
1900
|
-
} from "@workglow/util/schema";
|
|
1901
|
-
var inputSchema2 = {
|
|
1902
|
-
type: "object",
|
|
1903
|
-
properties: {
|
|
1904
|
-
doc_id: {
|
|
1905
|
-
type: "string",
|
|
1906
|
-
title: "Document ID",
|
|
1907
|
-
description: "The document ID"
|
|
1908
|
-
},
|
|
1909
|
-
doc_title: {
|
|
1910
|
-
type: "string",
|
|
1911
|
-
title: "Document Title",
|
|
1912
|
-
description: "Human-readable title for the source document"
|
|
1913
|
-
},
|
|
1914
|
-
chunks: {
|
|
1915
|
-
type: "array",
|
|
1916
|
-
items: ChunkRecordSchema(),
|
|
1917
|
-
title: "Chunks",
|
|
1918
|
-
description: "Array of chunk records"
|
|
1919
|
-
},
|
|
1920
|
-
vector: {
|
|
1921
|
-
type: "array",
|
|
1922
|
-
items: TypedArraySchema3({
|
|
1923
|
-
title: "Vector",
|
|
1924
|
-
description: "Vector embedding"
|
|
1925
|
-
}),
|
|
1926
|
-
title: "Vectors",
|
|
1927
|
-
description: "Embeddings from TextEmbeddingTask"
|
|
1928
|
-
}
|
|
1929
|
-
},
|
|
1930
|
-
required: ["chunks", "vector"],
|
|
1931
|
-
additionalProperties: false
|
|
1932
|
-
};
|
|
1933
|
-
var outputSchema2 = {
|
|
1934
|
-
type: "object",
|
|
1935
|
-
properties: {
|
|
1936
|
-
ids: {
|
|
1937
|
-
type: "array",
|
|
1938
|
-
items: { type: "string" },
|
|
1939
|
-
title: "IDs",
|
|
1940
|
-
description: "Chunk IDs for vector store"
|
|
1941
|
-
},
|
|
1942
|
-
vectors: {
|
|
1943
|
-
type: "array",
|
|
1944
|
-
items: TypedArraySchema3({
|
|
1945
|
-
title: "Vector",
|
|
1946
|
-
description: "Vector embedding"
|
|
1947
|
-
}),
|
|
1948
|
-
title: "Vectors",
|
|
1949
|
-
description: "Vector embeddings"
|
|
1950
|
-
},
|
|
1951
|
-
metadata: {
|
|
1952
|
-
type: "array",
|
|
1953
|
-
items: {
|
|
1954
|
-
type: "object",
|
|
1955
|
-
title: "Metadata",
|
|
1956
|
-
description: "Metadata for vector store"
|
|
1957
|
-
},
|
|
1958
|
-
title: "Metadata",
|
|
1959
|
-
description: "Metadata for each vector"
|
|
1960
|
-
},
|
|
1961
|
-
texts: {
|
|
1962
|
-
type: "array",
|
|
1963
|
-
items: { type: "string" },
|
|
1964
|
-
title: "Texts",
|
|
1965
|
-
description: "Chunk texts (for reference)"
|
|
1966
|
-
}
|
|
1967
|
-
},
|
|
1968
|
-
required: ["ids", "vectors", "metadata", "texts"],
|
|
1969
|
-
additionalProperties: false
|
|
1970
|
-
};
|
|
1971
|
-
|
|
1972
|
-
class ChunkToVectorTask extends Task3 {
|
|
1973
|
-
static type = "ChunkToVectorTask";
|
|
1974
|
-
static category = "Document";
|
|
1975
|
-
static title = "Chunk to Vector";
|
|
1976
|
-
static description = "Transform chunks and embeddings to vector store format";
|
|
1977
|
-
static cacheable = true;
|
|
1978
|
-
static inputSchema() {
|
|
1979
|
-
return inputSchema2;
|
|
1980
|
-
}
|
|
1981
|
-
static outputSchema() {
|
|
1982
|
-
return outputSchema2;
|
|
1983
|
-
}
|
|
1984
|
-
async execute(input, context) {
|
|
1985
|
-
const { chunks, vector, doc_title } = input;
|
|
1986
|
-
const chunkArray = chunks;
|
|
1987
|
-
if (!chunkArray || !vector) {
|
|
1988
|
-
throw new Error("Both chunks and vector are required");
|
|
1989
|
-
}
|
|
1990
|
-
if (chunkArray.length !== vector.length) {
|
|
1991
|
-
throw new Error(`Mismatch: ${chunkArray.length} chunks but ${vector.length} vectors`);
|
|
1992
|
-
}
|
|
1993
|
-
const ids = [];
|
|
1994
|
-
const metadata = [];
|
|
1995
|
-
const texts = [];
|
|
1996
|
-
for (let i = 0;i < chunkArray.length; i++) {
|
|
1997
|
-
const chunk = chunkArray[i];
|
|
1998
|
-
ids.push(chunk.chunkId);
|
|
1999
|
-
texts.push(chunk.text);
|
|
2000
|
-
metadata.push({
|
|
2001
|
-
doc_id: chunk.doc_id,
|
|
2002
|
-
chunkId: chunk.chunkId,
|
|
2003
|
-
leafNodeId: chunk.nodePath[chunk.nodePath.length - 1],
|
|
2004
|
-
depth: chunk.depth,
|
|
2005
|
-
text: chunk.text,
|
|
2006
|
-
nodePath: chunk.nodePath,
|
|
2007
|
-
...doc_title ? { doc_title } : {},
|
|
2008
|
-
...chunk.summary ? { summary: chunk.summary } : {},
|
|
2009
|
-
...chunk.entities ? { entities: chunk.entities } : {}
|
|
2010
|
-
});
|
|
2011
|
-
}
|
|
2012
|
-
return {
|
|
2013
|
-
ids,
|
|
2014
|
-
vectors: vector,
|
|
2015
|
-
metadata,
|
|
2016
|
-
texts
|
|
2017
|
-
};
|
|
2018
|
-
}
|
|
2019
|
-
}
|
|
2020
|
-
var chunkToVector = (input, config) => {
|
|
2021
|
-
return new ChunkToVectorTask(config).run(input);
|
|
2022
|
-
};
|
|
2023
|
-
Workflow4.prototype.chunkToVector = CreateWorkflow4(ChunkToVectorTask);
|
|
2024
|
-
|
|
2025
|
-
// src/task/ChunkVectorHybridSearchTask.ts
|
|
2026
|
-
import { TypeKnowledgeBase as TypeKnowledgeBase2 } from "@workglow/knowledge-base";
|
|
2027
|
-
import { CreateWorkflow as CreateWorkflow5, Task as Task4, Workflow as Workflow5 } from "@workglow/task-graph";
|
|
2028
|
-
import {
|
|
2029
|
-
TypedArraySchema as TypedArraySchema4
|
|
2030
|
-
} from "@workglow/util/schema";
|
|
2031
|
-
var inputSchema3 = {
|
|
2032
|
-
type: "object",
|
|
2033
|
-
properties: {
|
|
2034
|
-
knowledgeBase: TypeKnowledgeBase2({
|
|
2035
|
-
title: "Knowledge Base",
|
|
2036
|
-
description: "The knowledge base instance to search in (must support hybridSearch)"
|
|
2037
|
-
}),
|
|
2038
|
-
queryVector: TypedArraySchema4({
|
|
2039
|
-
title: "Query Vector",
|
|
2040
|
-
description: "The query vector for semantic search"
|
|
2041
|
-
}),
|
|
2042
|
-
queryText: {
|
|
2043
|
-
type: "string",
|
|
2044
|
-
title: "Query Text",
|
|
2045
|
-
description: "The query text for full-text search"
|
|
2046
|
-
},
|
|
2047
|
-
topK: {
|
|
2048
|
-
type: "number",
|
|
2049
|
-
title: "Top K",
|
|
2050
|
-
description: "Number of top results to return",
|
|
2051
|
-
minimum: 1,
|
|
2052
|
-
default: 10
|
|
2053
|
-
},
|
|
2054
|
-
filter: {
|
|
2055
|
-
type: "object",
|
|
2056
|
-
title: "Metadata Filter",
|
|
2057
|
-
description: "Filter results by metadata fields"
|
|
2058
|
-
},
|
|
2059
|
-
scoreThreshold: {
|
|
2060
|
-
type: "number",
|
|
2061
|
-
title: "Score Threshold",
|
|
2062
|
-
description: "Minimum combined score threshold (0-1)",
|
|
2063
|
-
minimum: 0,
|
|
2064
|
-
maximum: 1,
|
|
2065
|
-
default: 0
|
|
2066
|
-
},
|
|
2067
|
-
vectorWeight: {
|
|
2068
|
-
type: "number",
|
|
2069
|
-
title: "Vector Weight",
|
|
2070
|
-
description: "Weight for vector similarity (0-1), remainder goes to text relevance",
|
|
2071
|
-
minimum: 0,
|
|
2072
|
-
maximum: 1,
|
|
2073
|
-
default: 0.7
|
|
2074
|
-
},
|
|
2075
|
-
returnVectors: {
|
|
2076
|
-
type: "boolean",
|
|
2077
|
-
title: "Return Vectors",
|
|
2078
|
-
description: "Whether to return vector embeddings in results",
|
|
2079
|
-
default: false
|
|
2080
|
-
}
|
|
2081
|
-
},
|
|
2082
|
-
required: ["knowledgeBase", "queryVector", "queryText"],
|
|
2083
|
-
additionalProperties: false
|
|
2084
|
-
};
|
|
2085
|
-
var outputSchema3 = {
|
|
2086
|
-
type: "object",
|
|
2087
|
-
properties: {
|
|
2088
|
-
chunks: {
|
|
2089
|
-
type: "array",
|
|
2090
|
-
items: { type: "string" },
|
|
2091
|
-
title: "Text Chunks",
|
|
2092
|
-
description: "Retrieved text chunks"
|
|
2093
|
-
},
|
|
2094
|
-
chunk_ids: {
|
|
2095
|
-
type: "array",
|
|
2096
|
-
items: { type: "string" },
|
|
2097
|
-
title: "Chunk IDs",
|
|
2098
|
-
description: "IDs of retrieved chunks"
|
|
2099
|
-
},
|
|
2100
|
-
metadata: {
|
|
2101
|
-
type: "array",
|
|
2102
|
-
items: {
|
|
2103
|
-
type: "object",
|
|
2104
|
-
title: "Metadata",
|
|
2105
|
-
description: "Metadata of retrieved chunk"
|
|
2106
|
-
},
|
|
2107
|
-
title: "Metadata",
|
|
2108
|
-
description: "Metadata of retrieved chunks"
|
|
2109
|
-
},
|
|
2110
|
-
scores: {
|
|
2111
|
-
type: "array",
|
|
2112
|
-
items: { type: "number" },
|
|
2113
|
-
title: "Scores",
|
|
2114
|
-
description: "Combined relevance scores for each result"
|
|
2115
|
-
},
|
|
2116
|
-
vectors: {
|
|
2117
|
-
type: "array",
|
|
2118
|
-
items: TypedArraySchema4({
|
|
2119
|
-
title: "Vector",
|
|
2120
|
-
description: "Vector embedding"
|
|
2121
|
-
}),
|
|
2122
|
-
title: "Vectors",
|
|
2123
|
-
description: "Vector embeddings (if returnVectors is true)"
|
|
2124
|
-
},
|
|
2125
|
-
count: {
|
|
2126
|
-
type: "number",
|
|
2127
|
-
title: "Count",
|
|
2128
|
-
description: "Number of results returned"
|
|
2129
|
-
},
|
|
2130
|
-
query: {
|
|
2131
|
-
type: "string",
|
|
2132
|
-
title: "Query",
|
|
2133
|
-
description: "The text query used for search (pass-through)"
|
|
2134
|
-
}
|
|
2135
|
-
},
|
|
2136
|
-
required: ["chunks", "chunk_ids", "metadata", "scores", "count", "query"],
|
|
2137
|
-
additionalProperties: false
|
|
2138
|
-
};
|
|
2139
|
-
|
|
2140
|
-
class ChunkVectorHybridSearchTask extends Task4 {
|
|
2141
|
-
static type = "ChunkVectorHybridSearchTask";
|
|
2142
|
-
static category = "RAG";
|
|
2143
|
-
static title = "Hybrid Search";
|
|
2144
|
-
static description = "Combined vector + full-text search for improved retrieval";
|
|
2145
|
-
static cacheable = true;
|
|
2146
|
-
static inputSchema() {
|
|
2147
|
-
return inputSchema3;
|
|
2148
|
-
}
|
|
2149
|
-
static outputSchema() {
|
|
2150
|
-
return outputSchema3;
|
|
2151
|
-
}
|
|
2152
|
-
async execute(input, context) {
|
|
2153
|
-
const {
|
|
2154
|
-
knowledgeBase,
|
|
2155
|
-
queryVector,
|
|
2156
|
-
queryText,
|
|
2157
|
-
topK = 10,
|
|
2158
|
-
filter,
|
|
2159
|
-
scoreThreshold = 0,
|
|
2160
|
-
vectorWeight = 0.7,
|
|
2161
|
-
returnVectors = false
|
|
2162
|
-
} = input;
|
|
2163
|
-
const kb = knowledgeBase;
|
|
2164
1888
|
const searchVector = queryVector instanceof Float32Array ? queryVector : new Float32Array(queryVector);
|
|
2165
|
-
const results = await kb.hybridSearch(searchVector, {
|
|
1889
|
+
const results = method === "hybrid" ? await kb.hybridSearch(searchVector, {
|
|
2166
1890
|
textQuery: queryText,
|
|
2167
1891
|
topK,
|
|
2168
1892
|
filter,
|
|
2169
1893
|
scoreThreshold,
|
|
2170
1894
|
vectorWeight
|
|
1895
|
+
}) : await kb.similaritySearch(searchVector, {
|
|
1896
|
+
topK,
|
|
1897
|
+
filter,
|
|
1898
|
+
scoreThreshold
|
|
2171
1899
|
});
|
|
2172
1900
|
const chunks = results.map((r) => {
|
|
2173
1901
|
const meta = r.metadata;
|
|
@@ -2179,7 +1907,7 @@ class ChunkVectorHybridSearchTask extends Task4 {
|
|
|
2179
1907
|
metadata: results.map((r) => r.metadata),
|
|
2180
1908
|
scores: results.map((r) => r.score),
|
|
2181
1909
|
count: results.length,
|
|
2182
|
-
query
|
|
1910
|
+
query
|
|
2183
1911
|
};
|
|
2184
1912
|
if (returnVectors) {
|
|
2185
1913
|
output.vectors = results.map((r) => r.vector);
|
|
@@ -2187,163 +1915,39 @@ class ChunkVectorHybridSearchTask extends Task4 {
|
|
|
2187
1915
|
return output;
|
|
2188
1916
|
}
|
|
2189
1917
|
}
|
|
2190
|
-
var
|
|
2191
|
-
return new
|
|
2192
|
-
};
|
|
2193
|
-
Workflow5.prototype.hybridSearch = CreateWorkflow5(ChunkVectorHybridSearchTask);
|
|
2194
|
-
|
|
2195
|
-
// src/task/ChunkVectorSearchTask.ts
|
|
2196
|
-
import { TypeKnowledgeBase as TypeKnowledgeBase3 } from "@workglow/knowledge-base";
|
|
2197
|
-
import { CreateWorkflow as CreateWorkflow6, Task as Task5, Workflow as Workflow6 } from "@workglow/task-graph";
|
|
2198
|
-
import {
|
|
2199
|
-
TypedArraySchema as TypedArraySchema5
|
|
2200
|
-
} from "@workglow/util/schema";
|
|
2201
|
-
var inputSchema4 = {
|
|
2202
|
-
type: "object",
|
|
2203
|
-
properties: {
|
|
2204
|
-
knowledgeBase: TypeKnowledgeBase3({
|
|
2205
|
-
title: "Knowledge Base",
|
|
2206
|
-
description: "The knowledge base instance to search in"
|
|
2207
|
-
}),
|
|
2208
|
-
query: TypedArraySchema5({
|
|
2209
|
-
title: "Query Vector",
|
|
2210
|
-
description: "The query vector to search for similar vectors"
|
|
2211
|
-
}),
|
|
2212
|
-
topK: {
|
|
2213
|
-
type: "number",
|
|
2214
|
-
title: "Top K",
|
|
2215
|
-
description: "Number of top results to return",
|
|
2216
|
-
minimum: 1,
|
|
2217
|
-
default: 10
|
|
2218
|
-
},
|
|
2219
|
-
filter: {
|
|
2220
|
-
type: "object",
|
|
2221
|
-
title: "Metadata Filter",
|
|
2222
|
-
description: "Filter results by metadata fields"
|
|
2223
|
-
},
|
|
2224
|
-
scoreThreshold: {
|
|
2225
|
-
type: "number",
|
|
2226
|
-
title: "Score Threshold",
|
|
2227
|
-
description: "Minimum similarity score threshold (0-1)",
|
|
2228
|
-
minimum: 0,
|
|
2229
|
-
maximum: 1,
|
|
2230
|
-
default: 0
|
|
2231
|
-
}
|
|
2232
|
-
},
|
|
2233
|
-
required: ["knowledgeBase", "query"],
|
|
2234
|
-
additionalProperties: false
|
|
2235
|
-
};
|
|
2236
|
-
var outputSchema4 = {
|
|
2237
|
-
type: "object",
|
|
2238
|
-
properties: {
|
|
2239
|
-
ids: {
|
|
2240
|
-
type: "array",
|
|
2241
|
-
items: { type: "string" },
|
|
2242
|
-
title: "IDs",
|
|
2243
|
-
description: "IDs of matching vectors"
|
|
2244
|
-
},
|
|
2245
|
-
vectors: {
|
|
2246
|
-
type: "array",
|
|
2247
|
-
items: TypedArraySchema5({
|
|
2248
|
-
title: "Vector",
|
|
2249
|
-
description: "Matching vector embedding"
|
|
2250
|
-
}),
|
|
2251
|
-
title: "Vectors",
|
|
2252
|
-
description: "Matching vector embeddings"
|
|
2253
|
-
},
|
|
2254
|
-
metadata: {
|
|
2255
|
-
type: "array",
|
|
2256
|
-
items: {
|
|
2257
|
-
type: "object",
|
|
2258
|
-
title: "Metadata",
|
|
2259
|
-
description: "Metadata of matching vector"
|
|
2260
|
-
},
|
|
2261
|
-
title: "Metadata",
|
|
2262
|
-
description: "Metadata of matching vectors"
|
|
2263
|
-
},
|
|
2264
|
-
scores: {
|
|
2265
|
-
type: "array",
|
|
2266
|
-
items: { type: "number" },
|
|
2267
|
-
title: "Scores",
|
|
2268
|
-
description: "Similarity scores for each result"
|
|
2269
|
-
},
|
|
2270
|
-
count: {
|
|
2271
|
-
type: "number",
|
|
2272
|
-
title: "Count",
|
|
2273
|
-
description: "Number of results returned"
|
|
2274
|
-
}
|
|
2275
|
-
},
|
|
2276
|
-
required: ["ids", "vectors", "metadata", "scores", "count"],
|
|
2277
|
-
additionalProperties: false
|
|
2278
|
-
};
|
|
2279
|
-
|
|
2280
|
-
class ChunkVectorSearchTask extends Task5 {
|
|
2281
|
-
static type = "ChunkVectorSearchTask";
|
|
2282
|
-
static category = "Vector Store";
|
|
2283
|
-
static title = "Vector Store Search";
|
|
2284
|
-
static description = "Search for similar vectors in a knowledge base";
|
|
2285
|
-
static cacheable = true;
|
|
2286
|
-
static inputSchema() {
|
|
2287
|
-
return inputSchema4;
|
|
2288
|
-
}
|
|
2289
|
-
static outputSchema() {
|
|
2290
|
-
return outputSchema4;
|
|
2291
|
-
}
|
|
2292
|
-
async execute(input, _context) {
|
|
2293
|
-
const { knowledgeBase, query, topK = 10, filter, scoreThreshold = 0 } = input;
|
|
2294
|
-
const kb = knowledgeBase;
|
|
2295
|
-
const results = await kb.similaritySearch(query, {
|
|
2296
|
-
topK,
|
|
2297
|
-
filter,
|
|
2298
|
-
scoreThreshold
|
|
2299
|
-
});
|
|
2300
|
-
return {
|
|
2301
|
-
ids: results.map((r) => r.chunk_id),
|
|
2302
|
-
vectors: results.map((r) => r.vector),
|
|
2303
|
-
metadata: results.map((r) => r.metadata),
|
|
2304
|
-
scores: results.map((r) => r.score),
|
|
2305
|
-
count: results.length
|
|
2306
|
-
};
|
|
2307
|
-
}
|
|
2308
|
-
}
|
|
2309
|
-
var vectorStoreSearch = (input, config) => {
|
|
2310
|
-
return new ChunkVectorSearchTask(config).run(input);
|
|
1918
|
+
var chunkRetrieval = (input, config) => {
|
|
1919
|
+
return new ChunkRetrievalTask(config).run(input);
|
|
2311
1920
|
};
|
|
2312
|
-
|
|
1921
|
+
Workflow3.prototype.chunkRetrieval = CreateWorkflow3(ChunkRetrievalTask);
|
|
2313
1922
|
|
|
2314
1923
|
// src/task/ChunkVectorUpsertTask.ts
|
|
2315
|
-
import { TypeKnowledgeBase as
|
|
2316
|
-
import { CreateWorkflow as
|
|
1924
|
+
import { ChunkRecordArraySchema, TypeKnowledgeBase as TypeKnowledgeBase2 } from "@workglow/knowledge-base";
|
|
1925
|
+
import { CreateWorkflow as CreateWorkflow4, Task as Task3, Workflow as Workflow4 } from "@workglow/task-graph";
|
|
2317
1926
|
import {
|
|
2318
|
-
TypedArraySchema as
|
|
1927
|
+
TypedArraySchema as TypedArraySchema3
|
|
2319
1928
|
} from "@workglow/util/schema";
|
|
2320
|
-
var
|
|
1929
|
+
var inputSchema2 = {
|
|
2321
1930
|
type: "object",
|
|
2322
1931
|
properties: {
|
|
2323
|
-
knowledgeBase:
|
|
1932
|
+
knowledgeBase: TypeKnowledgeBase2({
|
|
2324
1933
|
title: "Knowledge Base",
|
|
2325
1934
|
description: "The knowledge base instance to store vectors in"
|
|
2326
1935
|
}),
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
title: "Document ID",
|
|
2330
|
-
description: "The document ID"
|
|
2331
|
-
},
|
|
2332
|
-
vectors: TypeSingleOrArray(TypedArraySchema6({
|
|
1936
|
+
chunks: ChunkRecordArraySchema,
|
|
1937
|
+
vector: TypeSingleOrArray(TypedArraySchema3({
|
|
2333
1938
|
title: "Vectors",
|
|
2334
|
-
description: "The vector embeddings"
|
|
1939
|
+
description: "The vector embeddings, aligned 1:1 with chunks"
|
|
2335
1940
|
})),
|
|
2336
|
-
|
|
2337
|
-
type: "
|
|
2338
|
-
title: "
|
|
2339
|
-
description: "
|
|
2340
|
-
|
|
2341
|
-
})
|
|
1941
|
+
doc_title: {
|
|
1942
|
+
type: "string",
|
|
1943
|
+
title: "Document Title",
|
|
1944
|
+
description: "Optional human-readable title stamped onto each chunk's metadata"
|
|
1945
|
+
}
|
|
2342
1946
|
},
|
|
2343
|
-
required: ["knowledgeBase", "
|
|
1947
|
+
required: ["knowledgeBase", "chunks", "vector"],
|
|
2344
1948
|
additionalProperties: false
|
|
2345
1949
|
};
|
|
2346
|
-
var
|
|
1950
|
+
var outputSchema2 = {
|
|
2347
1951
|
type: "object",
|
|
2348
1952
|
properties: {
|
|
2349
1953
|
count: {
|
|
@@ -2354,7 +1958,7 @@ var outputSchema5 = {
|
|
|
2354
1958
|
doc_id: {
|
|
2355
1959
|
type: "string",
|
|
2356
1960
|
title: "Document ID",
|
|
2357
|
-
description: "The document ID"
|
|
1961
|
+
description: "The document ID (read from the first chunk)"
|
|
2358
1962
|
},
|
|
2359
1963
|
chunk_ids: {
|
|
2360
1964
|
type: "array",
|
|
@@ -2367,74 +1971,73 @@ var outputSchema5 = {
|
|
|
2367
1971
|
additionalProperties: false
|
|
2368
1972
|
};
|
|
2369
1973
|
|
|
2370
|
-
class ChunkVectorUpsertTask extends
|
|
1974
|
+
class ChunkVectorUpsertTask extends Task3 {
|
|
2371
1975
|
static type = "ChunkVectorUpsertTask";
|
|
2372
1976
|
static category = "Vector Store";
|
|
2373
1977
|
static title = "Add to Vector Store";
|
|
2374
|
-
static description = "Store
|
|
1978
|
+
static description = "Store chunks + their embeddings in a knowledge base (1:1 aligned)";
|
|
2375
1979
|
static cacheable = false;
|
|
2376
1980
|
static inputSchema() {
|
|
2377
|
-
return
|
|
1981
|
+
return inputSchema2;
|
|
2378
1982
|
}
|
|
2379
1983
|
static outputSchema() {
|
|
2380
|
-
return
|
|
1984
|
+
return outputSchema2;
|
|
2381
1985
|
}
|
|
2382
1986
|
async execute(input, context) {
|
|
2383
|
-
const { knowledgeBase,
|
|
2384
|
-
const
|
|
2385
|
-
const
|
|
1987
|
+
const { knowledgeBase, chunks, vector, doc_title } = input;
|
|
1988
|
+
const chunkArray = chunks;
|
|
1989
|
+
const vectorArray = Array.isArray(vector) ? vector : [vector];
|
|
1990
|
+
if (chunkArray.length !== vectorArray.length) {
|
|
1991
|
+
throw new Error(`Mismatch: ${chunkArray.length} chunks but ${vectorArray.length} vectors \u2014 they must be 1:1 aligned`);
|
|
1992
|
+
}
|
|
1993
|
+
if (chunkArray.length === 0) {
|
|
1994
|
+
return { doc_id: "", chunk_ids: [], count: 0 };
|
|
1995
|
+
}
|
|
2386
1996
|
const kb = knowledgeBase;
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
if (
|
|
2390
|
-
throw new Error(
|
|
1997
|
+
const doc_id = chunkArray[0].doc_id;
|
|
1998
|
+
for (let i = 1;i < chunkArray.length; i++) {
|
|
1999
|
+
if (chunkArray[i].doc_id !== doc_id) {
|
|
2000
|
+
throw new Error(`ChunkVectorUpsertTask expects all chunks to share one doc_id, but chunk[${i}] has doc_id=${JSON.stringify(chunkArray[i].doc_id)} (expected ${JSON.stringify(doc_id)}).`);
|
|
2391
2001
|
}
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
}
|
|
2399
|
-
|
|
2400
|
-
const results = await kb.upsertChunksBulk(entities);
|
|
2401
|
-
const chunk_ids = results.map((r) => r.chunk_id);
|
|
2402
|
-
return {
|
|
2403
|
-
doc_id,
|
|
2404
|
-
chunk_ids,
|
|
2405
|
-
count: chunk_ids.length
|
|
2002
|
+
}
|
|
2003
|
+
await context.updateProgress(1, "Upserting vectors");
|
|
2004
|
+
const entities = chunkArray.map((chunk, i) => {
|
|
2005
|
+
const leafNodeId = chunk.leafNodeId ?? chunk.nodePath[chunk.nodePath.length - 1] ?? undefined;
|
|
2006
|
+
const metadata = {
|
|
2007
|
+
...chunk,
|
|
2008
|
+
...leafNodeId !== undefined ? { leafNodeId } : {},
|
|
2009
|
+
...doc_title ? { doc_title } : {}
|
|
2406
2010
|
};
|
|
2407
|
-
} else if (vectorArray.length === 1) {
|
|
2408
|
-
const metadataItem = metadataArray[0];
|
|
2409
|
-
const result = await kb.upsertChunk({
|
|
2410
|
-
doc_id,
|
|
2411
|
-
vector: vectorArray[0],
|
|
2412
|
-
metadata: metadataItem
|
|
2413
|
-
});
|
|
2414
2011
|
return {
|
|
2415
|
-
doc_id,
|
|
2416
|
-
|
|
2417
|
-
|
|
2012
|
+
doc_id: chunk.doc_id,
|
|
2013
|
+
vector: vectorArray[i],
|
|
2014
|
+
metadata
|
|
2418
2015
|
};
|
|
2419
|
-
}
|
|
2016
|
+
});
|
|
2017
|
+
const results = await kb.upsertChunksBulk(entities);
|
|
2018
|
+
const chunk_ids = results.map((r) => r.chunk_id);
|
|
2420
2019
|
return {
|
|
2421
2020
|
doc_id,
|
|
2422
|
-
chunk_ids
|
|
2423
|
-
count:
|
|
2021
|
+
chunk_ids,
|
|
2022
|
+
count: chunk_ids.length
|
|
2424
2023
|
};
|
|
2425
2024
|
}
|
|
2426
2025
|
}
|
|
2427
2026
|
var chunkVectorUpsert = (input, config) => {
|
|
2428
2027
|
return new ChunkVectorUpsertTask(config).run(input);
|
|
2429
2028
|
};
|
|
2430
|
-
|
|
2029
|
+
Workflow4.prototype.chunkVectorUpsert = CreateWorkflow4(ChunkVectorUpsertTask);
|
|
2431
2030
|
|
|
2432
2031
|
// src/task/ContextBuilderTask.ts
|
|
2433
2032
|
import { estimateTokens } from "@workglow/knowledge-base";
|
|
2434
|
-
import {
|
|
2033
|
+
import {
|
|
2034
|
+
CreateWorkflow as CreateWorkflow6,
|
|
2035
|
+
Task as Task4,
|
|
2036
|
+
Workflow as Workflow6
|
|
2037
|
+
} from "@workglow/task-graph";
|
|
2435
2038
|
|
|
2436
2039
|
// src/task/CountTokensTask.ts
|
|
2437
|
-
import { CreateWorkflow as
|
|
2040
|
+
import { CreateWorkflow as CreateWorkflow5, Workflow as Workflow5 } from "@workglow/task-graph";
|
|
2438
2041
|
var modelSchema4 = TypeModel("model");
|
|
2439
2042
|
var CountTokensInputSchema = {
|
|
2440
2043
|
type: "object",
|
|
@@ -2478,7 +2081,7 @@ class CountTokensTask extends AiTask {
|
|
|
2478
2081
|
var countTokens = async (input, config) => {
|
|
2479
2082
|
return new CountTokensTask(config).run(input);
|
|
2480
2083
|
};
|
|
2481
|
-
|
|
2084
|
+
Workflow5.prototype.countTokens = CreateWorkflow5(CountTokensTask);
|
|
2482
2085
|
|
|
2483
2086
|
// src/task/ContextBuilderTask.ts
|
|
2484
2087
|
var ContextFormat = {
|
|
@@ -2492,7 +2095,7 @@ var modelSchema5 = TypeModel("model", {
|
|
|
2492
2095
|
title: "Model",
|
|
2493
2096
|
description: "Model to use for token counting (optional, falls back to estimation)"
|
|
2494
2097
|
});
|
|
2495
|
-
var
|
|
2098
|
+
var inputSchema3 = {
|
|
2496
2099
|
type: "object",
|
|
2497
2100
|
properties: {
|
|
2498
2101
|
chunks: {
|
|
@@ -2557,7 +2160,7 @@ var inputSchema6 = {
|
|
|
2557
2160
|
required: ["chunks"],
|
|
2558
2161
|
additionalProperties: false
|
|
2559
2162
|
};
|
|
2560
|
-
var
|
|
2163
|
+
var outputSchema3 = {
|
|
2561
2164
|
type: "object",
|
|
2562
2165
|
properties: {
|
|
2563
2166
|
context: {
|
|
@@ -2585,19 +2188,22 @@ var outputSchema6 = {
|
|
|
2585
2188
|
additionalProperties: false
|
|
2586
2189
|
};
|
|
2587
2190
|
|
|
2588
|
-
class ContextBuilderTask extends
|
|
2191
|
+
class ContextBuilderTask extends Task4 {
|
|
2589
2192
|
static type = "ContextBuilderTask";
|
|
2590
2193
|
static category = "RAG";
|
|
2591
2194
|
static title = "Context Builder";
|
|
2592
2195
|
static description = "Format retrieved chunks into context for LLM prompts";
|
|
2593
2196
|
static cacheable = true;
|
|
2594
2197
|
static inputSchema() {
|
|
2595
|
-
return
|
|
2198
|
+
return inputSchema3;
|
|
2596
2199
|
}
|
|
2597
2200
|
static outputSchema() {
|
|
2598
|
-
return
|
|
2201
|
+
return outputSchema3;
|
|
2202
|
+
}
|
|
2203
|
+
async execute(input, context) {
|
|
2204
|
+
return this.executePreview(input, context);
|
|
2599
2205
|
}
|
|
2600
|
-
async
|
|
2206
|
+
async executePreview(input, context) {
|
|
2601
2207
|
const {
|
|
2602
2208
|
chunks,
|
|
2603
2209
|
metadata = [],
|
|
@@ -2778,14 +2384,14 @@ class ContextBuilderTask extends Task7 {
|
|
|
2778
2384
|
var contextBuilder = (input, config) => {
|
|
2779
2385
|
return new ContextBuilderTask(config).run(input);
|
|
2780
2386
|
};
|
|
2781
|
-
|
|
2387
|
+
Workflow6.prototype.contextBuilder = CreateWorkflow6(ContextBuilderTask);
|
|
2782
2388
|
|
|
2783
2389
|
// src/task/DocumentEnricherTask.ts
|
|
2784
2390
|
import { getChildren, hasChildren } from "@workglow/knowledge-base";
|
|
2785
|
-
import { CreateWorkflow as
|
|
2391
|
+
import { CreateWorkflow as CreateWorkflow9, Task as Task5, Workflow as Workflow9 } from "@workglow/task-graph";
|
|
2786
2392
|
|
|
2787
2393
|
// src/task/TextNamedEntityRecognitionTask.ts
|
|
2788
|
-
import { CreateWorkflow as
|
|
2394
|
+
import { CreateWorkflow as CreateWorkflow7, Workflow as Workflow7 } from "@workglow/task-graph";
|
|
2789
2395
|
var modelSchema6 = TypeModel("model:TextNamedEntityRecognitionTask");
|
|
2790
2396
|
var TextNamedEntityRecognitionInputSchema = {
|
|
2791
2397
|
type: "object",
|
|
@@ -2860,10 +2466,10 @@ class TextNamedEntityRecognitionTask extends AiTask {
|
|
|
2860
2466
|
var textNamedEntityRecognition = (input, config) => {
|
|
2861
2467
|
return new TextNamedEntityRecognitionTask(config).run(input);
|
|
2862
2468
|
};
|
|
2863
|
-
|
|
2469
|
+
Workflow7.prototype.textNamedEntityRecognition = CreateWorkflow7(TextNamedEntityRecognitionTask);
|
|
2864
2470
|
|
|
2865
2471
|
// src/task/TextSummaryTask.ts
|
|
2866
|
-
import { CreateWorkflow as
|
|
2472
|
+
import { CreateWorkflow as CreateWorkflow8, Workflow as Workflow8 } from "@workglow/task-graph";
|
|
2867
2473
|
var modelSchema7 = TypeModel("model:TextSummaryTask");
|
|
2868
2474
|
var TextSummaryInputSchema = {
|
|
2869
2475
|
type: "object",
|
|
@@ -2907,10 +2513,10 @@ class TextSummaryTask extends StreamingAiTask {
|
|
|
2907
2513
|
var textSummary = async (input, config) => {
|
|
2908
2514
|
return new TextSummaryTask(config).run(input);
|
|
2909
2515
|
};
|
|
2910
|
-
|
|
2516
|
+
Workflow8.prototype.textSummary = CreateWorkflow8(TextSummaryTask);
|
|
2911
2517
|
|
|
2912
2518
|
// src/task/DocumentEnricherTask.ts
|
|
2913
|
-
var
|
|
2519
|
+
var inputSchema4 = {
|
|
2914
2520
|
type: "object",
|
|
2915
2521
|
properties: {
|
|
2916
2522
|
doc_id: {
|
|
@@ -2954,7 +2560,7 @@ var inputSchema7 = {
|
|
|
2954
2560
|
required: [],
|
|
2955
2561
|
additionalProperties: false
|
|
2956
2562
|
};
|
|
2957
|
-
var
|
|
2563
|
+
var outputSchema4 = {
|
|
2958
2564
|
type: "object",
|
|
2959
2565
|
properties: {
|
|
2960
2566
|
doc_id: {
|
|
@@ -2981,17 +2587,17 @@ var outputSchema7 = {
|
|
|
2981
2587
|
additionalProperties: false
|
|
2982
2588
|
};
|
|
2983
2589
|
|
|
2984
|
-
class DocumentEnricherTask extends
|
|
2590
|
+
class DocumentEnricherTask extends Task5 {
|
|
2985
2591
|
static type = "DocumentEnricherTask";
|
|
2986
2592
|
static category = "Document";
|
|
2987
2593
|
static title = "Document Enricher";
|
|
2988
2594
|
static description = "Enrich document nodes with summaries and entities";
|
|
2989
2595
|
static cacheable = true;
|
|
2990
2596
|
static inputSchema() {
|
|
2991
|
-
return
|
|
2597
|
+
return inputSchema4;
|
|
2992
2598
|
}
|
|
2993
2599
|
static outputSchema() {
|
|
2994
|
-
return
|
|
2600
|
+
return outputSchema4;
|
|
2995
2601
|
}
|
|
2996
2602
|
async execute(input, context) {
|
|
2997
2603
|
const {
|
|
@@ -3140,16 +2746,16 @@ class DocumentEnricherTask extends Task8 {
|
|
|
3140
2746
|
var documentEnricher = (input, config) => {
|
|
3141
2747
|
return new DocumentEnricherTask(config).run(input);
|
|
3142
2748
|
};
|
|
3143
|
-
|
|
2749
|
+
Workflow9.prototype.documentEnricher = CreateWorkflow9(DocumentEnricherTask);
|
|
3144
2750
|
|
|
3145
2751
|
// src/task/DocumentUpsertTask.ts
|
|
3146
|
-
import { Document, TypeKnowledgeBase as
|
|
2752
|
+
import { Document, TypeKnowledgeBase as TypeKnowledgeBase3 } from "@workglow/knowledge-base";
|
|
3147
2753
|
import { DocumentMetadataSchema } from "@workglow/knowledge-base";
|
|
3148
|
-
import { CreateWorkflow as
|
|
3149
|
-
var
|
|
2754
|
+
import { CreateWorkflow as CreateWorkflow10, Task as Task6, Workflow as Workflow10 } from "@workglow/task-graph";
|
|
2755
|
+
var inputSchema5 = {
|
|
3150
2756
|
type: "object",
|
|
3151
2757
|
properties: {
|
|
3152
|
-
knowledgeBase:
|
|
2758
|
+
knowledgeBase: TypeKnowledgeBase3({
|
|
3153
2759
|
title: "Knowledge Base",
|
|
3154
2760
|
description: "The knowledge base instance to store the document in"
|
|
3155
2761
|
}),
|
|
@@ -3177,7 +2783,7 @@ var inputSchema8 = {
|
|
|
3177
2783
|
required: ["knowledgeBase", "doc_id", "documentTree"],
|
|
3178
2784
|
additionalProperties: false
|
|
3179
2785
|
};
|
|
3180
|
-
var
|
|
2786
|
+
var outputSchema5 = {
|
|
3181
2787
|
type: "object",
|
|
3182
2788
|
properties: {
|
|
3183
2789
|
doc_id: {
|
|
@@ -3190,17 +2796,17 @@ var outputSchema8 = {
|
|
|
3190
2796
|
additionalProperties: false
|
|
3191
2797
|
};
|
|
3192
2798
|
|
|
3193
|
-
class DocumentUpsertTask extends
|
|
2799
|
+
class DocumentUpsertTask extends Task6 {
|
|
3194
2800
|
static type = "DocumentUpsertTask";
|
|
3195
2801
|
static category = "Vector Store";
|
|
3196
2802
|
static title = "Add Document";
|
|
3197
2803
|
static description = "Persist a parsed document tree to a knowledge base";
|
|
3198
2804
|
static cacheable = false;
|
|
3199
2805
|
static inputSchema() {
|
|
3200
|
-
return
|
|
2806
|
+
return inputSchema5;
|
|
3201
2807
|
}
|
|
3202
2808
|
static outputSchema() {
|
|
3203
|
-
return
|
|
2809
|
+
return outputSchema5;
|
|
3204
2810
|
}
|
|
3205
2811
|
async execute(input, context) {
|
|
3206
2812
|
const { knowledgeBase, doc_id, documentTree, title, metadata } = input;
|
|
@@ -3223,10 +2829,10 @@ class DocumentUpsertTask extends Task9 {
|
|
|
3223
2829
|
var documentUpsert = (input, config) => {
|
|
3224
2830
|
return new DocumentUpsertTask(config).run(input);
|
|
3225
2831
|
};
|
|
3226
|
-
|
|
2832
|
+
Workflow10.prototype.documentUpsert = CreateWorkflow10(DocumentUpsertTask);
|
|
3227
2833
|
|
|
3228
2834
|
// src/task/DownloadModelTask.ts
|
|
3229
|
-
import { CreateWorkflow as
|
|
2835
|
+
import { CreateWorkflow as CreateWorkflow11, Workflow as Workflow11 } from "@workglow/task-graph";
|
|
3230
2836
|
var modelSchema8 = TypeModel("model");
|
|
3231
2837
|
var DownloadModelInputSchema = {
|
|
3232
2838
|
type: "object",
|
|
@@ -3293,10 +2899,10 @@ class DownloadModelTask extends AiTask {
|
|
|
3293
2899
|
var downloadModel = (input, config) => {
|
|
3294
2900
|
return new DownloadModelTask(config).run(input);
|
|
3295
2901
|
};
|
|
3296
|
-
|
|
2902
|
+
Workflow11.prototype.downloadModel = CreateWorkflow11(DownloadModelTask);
|
|
3297
2903
|
|
|
3298
2904
|
// src/task/FaceDetectorTask.ts
|
|
3299
|
-
import { CreateWorkflow as
|
|
2905
|
+
import { CreateWorkflow as CreateWorkflow12, Workflow as Workflow12 } from "@workglow/task-graph";
|
|
3300
2906
|
var modelSchema9 = TypeModel("model:FaceDetectorTask");
|
|
3301
2907
|
var TypeBoundingBox2 = {
|
|
3302
2908
|
type: "object",
|
|
@@ -3424,10 +3030,10 @@ class FaceDetectorTask extends AiVisionTask {
|
|
|
3424
3030
|
var faceDetector = (input, config) => {
|
|
3425
3031
|
return new FaceDetectorTask(config).run(input);
|
|
3426
3032
|
};
|
|
3427
|
-
|
|
3033
|
+
Workflow12.prototype.faceDetector = CreateWorkflow12(FaceDetectorTask);
|
|
3428
3034
|
|
|
3429
3035
|
// src/task/FaceLandmarkerTask.ts
|
|
3430
|
-
import { CreateWorkflow as
|
|
3036
|
+
import { CreateWorkflow as CreateWorkflow13, Workflow as Workflow13 } from "@workglow/task-graph";
|
|
3431
3037
|
var modelSchema10 = TypeModel("model:FaceLandmarkerTask");
|
|
3432
3038
|
var TypeLandmark = {
|
|
3433
3039
|
type: "object",
|
|
@@ -3586,10 +3192,10 @@ class FaceLandmarkerTask extends AiVisionTask {
|
|
|
3586
3192
|
var faceLandmarker = (input, config) => {
|
|
3587
3193
|
return new FaceLandmarkerTask(config).run(input);
|
|
3588
3194
|
};
|
|
3589
|
-
|
|
3195
|
+
Workflow13.prototype.faceLandmarker = CreateWorkflow13(FaceLandmarkerTask);
|
|
3590
3196
|
|
|
3591
3197
|
// src/task/GestureRecognizerTask.ts
|
|
3592
|
-
import { CreateWorkflow as
|
|
3198
|
+
import { CreateWorkflow as CreateWorkflow14, Workflow as Workflow14 } from "@workglow/task-graph";
|
|
3593
3199
|
var modelSchema11 = TypeModel("model:GestureRecognizerTask");
|
|
3594
3200
|
var TypeLandmark2 = {
|
|
3595
3201
|
type: "object",
|
|
@@ -3754,10 +3360,10 @@ class GestureRecognizerTask extends AiVisionTask {
|
|
|
3754
3360
|
var gestureRecognizer = (input, config) => {
|
|
3755
3361
|
return new GestureRecognizerTask(config).run(input);
|
|
3756
3362
|
};
|
|
3757
|
-
|
|
3363
|
+
Workflow14.prototype.gestureRecognizer = CreateWorkflow14(GestureRecognizerTask);
|
|
3758
3364
|
|
|
3759
3365
|
// src/task/HandLandmarkerTask.ts
|
|
3760
|
-
import { CreateWorkflow as
|
|
3366
|
+
import { CreateWorkflow as CreateWorkflow15, Workflow as Workflow15 } from "@workglow/task-graph";
|
|
3761
3367
|
var modelSchema12 = TypeModel("model:HandLandmarkerTask");
|
|
3762
3368
|
var TypeLandmark3 = {
|
|
3763
3369
|
type: "object",
|
|
@@ -3899,22 +3505,22 @@ class HandLandmarkerTask extends AiVisionTask {
|
|
|
3899
3505
|
var handLandmarker = (input, config) => {
|
|
3900
3506
|
return new HandLandmarkerTask(config).run(input);
|
|
3901
3507
|
};
|
|
3902
|
-
|
|
3508
|
+
Workflow15.prototype.handLandmarker = CreateWorkflow15(HandLandmarkerTask);
|
|
3903
3509
|
|
|
3904
3510
|
// src/task/HierarchicalChunkerTask.ts
|
|
3905
3511
|
import {
|
|
3906
|
-
ChunkRecordSchema
|
|
3512
|
+
ChunkRecordSchema,
|
|
3907
3513
|
estimateTokens as estimateTokens2,
|
|
3908
3514
|
getChildren as getChildren2,
|
|
3909
3515
|
hasChildren as hasChildren2
|
|
3910
3516
|
} from "@workglow/knowledge-base";
|
|
3911
|
-
import { CreateWorkflow as
|
|
3517
|
+
import { CreateWorkflow as CreateWorkflow16, Task as Task7, Workflow as Workflow16 } from "@workglow/task-graph";
|
|
3912
3518
|
import { uuid4 } from "@workglow/util";
|
|
3913
3519
|
var modelSchema13 = TypeModel("model", {
|
|
3914
3520
|
title: "Model",
|
|
3915
3521
|
description: "Model to use for token counting"
|
|
3916
3522
|
});
|
|
3917
|
-
var
|
|
3523
|
+
var inputSchema6 = {
|
|
3918
3524
|
type: "object",
|
|
3919
3525
|
properties: {
|
|
3920
3526
|
doc_id: {
|
|
@@ -3961,7 +3567,7 @@ var inputSchema9 = {
|
|
|
3961
3567
|
required: ["doc_id", "documentTree"],
|
|
3962
3568
|
additionalProperties: false
|
|
3963
3569
|
};
|
|
3964
|
-
var
|
|
3570
|
+
var outputSchema6 = {
|
|
3965
3571
|
type: "object",
|
|
3966
3572
|
properties: {
|
|
3967
3573
|
doc_id: {
|
|
@@ -3971,7 +3577,7 @@ var outputSchema9 = {
|
|
|
3971
3577
|
},
|
|
3972
3578
|
chunks: {
|
|
3973
3579
|
type: "array",
|
|
3974
|
-
items:
|
|
3580
|
+
items: ChunkRecordSchema(),
|
|
3975
3581
|
title: "Chunks",
|
|
3976
3582
|
description: "Array of chunk records"
|
|
3977
3583
|
},
|
|
@@ -3991,17 +3597,17 @@ var outputSchema9 = {
|
|
|
3991
3597
|
additionalProperties: false
|
|
3992
3598
|
};
|
|
3993
3599
|
|
|
3994
|
-
class HierarchicalChunkerTask extends
|
|
3600
|
+
class HierarchicalChunkerTask extends Task7 {
|
|
3995
3601
|
static type = "HierarchicalChunkerTask";
|
|
3996
3602
|
static category = "Document";
|
|
3997
3603
|
static title = "Hierarchical Chunker";
|
|
3998
3604
|
static description = "Chunk documents hierarchically respecting token budgets";
|
|
3999
3605
|
static cacheable = true;
|
|
4000
3606
|
static inputSchema() {
|
|
4001
|
-
return
|
|
3607
|
+
return inputSchema6;
|
|
4002
3608
|
}
|
|
4003
3609
|
static outputSchema() {
|
|
4004
|
-
return
|
|
3610
|
+
return outputSchema6;
|
|
4005
3611
|
}
|
|
4006
3612
|
async execute(input, context) {
|
|
4007
3613
|
const {
|
|
@@ -4135,36 +3741,36 @@ class HierarchicalChunkerTask extends Task10 {
|
|
|
4135
3741
|
var hierarchicalChunker = (input, config) => {
|
|
4136
3742
|
return new HierarchicalChunkerTask(config).run(input);
|
|
4137
3743
|
};
|
|
4138
|
-
|
|
3744
|
+
Workflow16.prototype.hierarchicalChunker = CreateWorkflow16(HierarchicalChunkerTask);
|
|
4139
3745
|
|
|
4140
3746
|
// src/task/HierarchyJoinTask.ts
|
|
4141
|
-
import { ChunkRecordArraySchema, TypeKnowledgeBase as
|
|
4142
|
-
import { CreateWorkflow as
|
|
4143
|
-
var
|
|
3747
|
+
import { ChunkRecordArraySchema as ChunkRecordArraySchema2, TypeKnowledgeBase as TypeKnowledgeBase4 } from "@workglow/knowledge-base";
|
|
3748
|
+
import { CreateWorkflow as CreateWorkflow17, Task as Task8, Workflow as Workflow17 } from "@workglow/task-graph";
|
|
3749
|
+
var inputSchema7 = {
|
|
4144
3750
|
type: "object",
|
|
4145
3751
|
properties: {
|
|
4146
|
-
knowledgeBase:
|
|
3752
|
+
knowledgeBase: TypeKnowledgeBase4({
|
|
4147
3753
|
title: "Knowledge Base",
|
|
4148
3754
|
description: "The knowledge base to query for hierarchy"
|
|
4149
3755
|
}),
|
|
3756
|
+
metadata: ChunkRecordArraySchema2,
|
|
4150
3757
|
chunks: {
|
|
4151
3758
|
type: "array",
|
|
4152
3759
|
items: { type: "string" },
|
|
4153
3760
|
title: "Chunks",
|
|
4154
|
-
description: "Retrieved text chunks"
|
|
3761
|
+
description: "Retrieved text chunks (pass-through)"
|
|
4155
3762
|
},
|
|
4156
3763
|
chunk_ids: {
|
|
4157
3764
|
type: "array",
|
|
4158
3765
|
items: { type: "string" },
|
|
4159
3766
|
title: "Chunk IDs",
|
|
4160
|
-
description: "IDs of retrieved chunks"
|
|
3767
|
+
description: "IDs of retrieved chunks (pass-through)"
|
|
4161
3768
|
},
|
|
4162
|
-
metadata: ChunkRecordArraySchema,
|
|
4163
3769
|
scores: {
|
|
4164
3770
|
type: "array",
|
|
4165
3771
|
items: { type: "number" },
|
|
4166
3772
|
title: "Scores",
|
|
4167
|
-
description: "Similarity scores
|
|
3773
|
+
description: "Similarity scores (pass-through)"
|
|
4168
3774
|
},
|
|
4169
3775
|
includeParentSummaries: {
|
|
4170
3776
|
type: "boolean",
|
|
@@ -4179,74 +3785,72 @@ var inputSchema10 = {
|
|
|
4179
3785
|
default: true
|
|
4180
3786
|
}
|
|
4181
3787
|
},
|
|
4182
|
-
required: ["knowledgeBase", "
|
|
3788
|
+
required: ["knowledgeBase", "metadata"],
|
|
4183
3789
|
additionalProperties: false
|
|
4184
3790
|
};
|
|
4185
|
-
var
|
|
3791
|
+
var outputSchema7 = {
|
|
4186
3792
|
type: "object",
|
|
4187
3793
|
properties: {
|
|
3794
|
+
metadata: ChunkRecordArraySchema2,
|
|
4188
3795
|
chunks: {
|
|
4189
3796
|
type: "array",
|
|
4190
3797
|
items: { type: "string" },
|
|
4191
3798
|
title: "Chunks",
|
|
4192
|
-
description: "Retrieved text chunks"
|
|
3799
|
+
description: "Retrieved text chunks (pass-through)"
|
|
4193
3800
|
},
|
|
4194
3801
|
chunk_ids: {
|
|
4195
3802
|
type: "array",
|
|
4196
3803
|
items: { type: "string" },
|
|
4197
3804
|
title: "Chunk IDs",
|
|
4198
|
-
description: "IDs of retrieved chunks"
|
|
3805
|
+
description: "IDs of retrieved chunks (pass-through)"
|
|
4199
3806
|
},
|
|
4200
|
-
metadata: ChunkRecordArraySchema,
|
|
4201
3807
|
scores: {
|
|
4202
3808
|
type: "array",
|
|
4203
3809
|
items: { type: "number" },
|
|
4204
3810
|
title: "Scores",
|
|
4205
|
-
description: "Similarity scores"
|
|
3811
|
+
description: "Similarity scores (pass-through)"
|
|
4206
3812
|
},
|
|
4207
3813
|
count: {
|
|
4208
3814
|
type: "number",
|
|
4209
3815
|
title: "Count",
|
|
4210
|
-
description: "Number of
|
|
3816
|
+
description: "Number of enriched records"
|
|
4211
3817
|
}
|
|
4212
3818
|
},
|
|
4213
|
-
required: ["
|
|
3819
|
+
required: ["metadata", "count"],
|
|
4214
3820
|
additionalProperties: false
|
|
4215
3821
|
};
|
|
4216
3822
|
|
|
4217
|
-
class HierarchyJoinTask extends
|
|
3823
|
+
class HierarchyJoinTask extends Task8 {
|
|
4218
3824
|
static type = "HierarchyJoinTask";
|
|
4219
3825
|
static category = "RAG";
|
|
4220
3826
|
static title = "Hierarchy Join";
|
|
4221
|
-
static description = "Enrich
|
|
3827
|
+
static description = "Enrich retrieval metadata with document hierarchy context";
|
|
4222
3828
|
static cacheable = false;
|
|
4223
3829
|
static inputSchema() {
|
|
4224
|
-
return
|
|
3830
|
+
return inputSchema7;
|
|
4225
3831
|
}
|
|
4226
3832
|
static outputSchema() {
|
|
4227
|
-
return
|
|
3833
|
+
return outputSchema7;
|
|
4228
3834
|
}
|
|
4229
3835
|
async execute(input, context) {
|
|
4230
3836
|
const {
|
|
4231
3837
|
knowledgeBase,
|
|
3838
|
+
metadata,
|
|
4232
3839
|
chunks,
|
|
4233
3840
|
chunk_ids,
|
|
4234
|
-
metadata,
|
|
4235
3841
|
scores,
|
|
4236
3842
|
includeParentSummaries = true,
|
|
4237
3843
|
includeEntities = true
|
|
4238
3844
|
} = input;
|
|
4239
3845
|
const kb = knowledgeBase;
|
|
4240
3846
|
const enrichedMetadata = [];
|
|
4241
|
-
for (
|
|
4242
|
-
const chunkId = chunk_ids[i];
|
|
4243
|
-
const originalMetadata = metadata[i];
|
|
3847
|
+
for (const originalMetadata of metadata) {
|
|
4244
3848
|
if (!originalMetadata) {
|
|
4245
3849
|
enrichedMetadata.push({});
|
|
4246
3850
|
continue;
|
|
4247
3851
|
}
|
|
4248
3852
|
const doc_id = originalMetadata.doc_id;
|
|
4249
|
-
const leafNodeId = originalMetadata.leafNodeId;
|
|
3853
|
+
const leafNodeId = originalMetadata.leafNodeId ?? originalMetadata.nodePath?.[originalMetadata.nodePath.length - 1];
|
|
4250
3854
|
if (!doc_id || !leafNodeId) {
|
|
4251
3855
|
enrichedMetadata.push(originalMetadata);
|
|
4252
3856
|
continue;
|
|
@@ -4292,31 +3896,35 @@ class HierarchyJoinTask extends Task11 {
|
|
|
4292
3896
|
}
|
|
4293
3897
|
enrichedMetadata.push(enriched);
|
|
4294
3898
|
} catch (error) {
|
|
4295
|
-
console.error(`Failed to join hierarchy for chunk ${chunkId}:`, error);
|
|
3899
|
+
console.error(`Failed to join hierarchy for chunk ${originalMetadata.chunkId}:`, error);
|
|
4296
3900
|
enrichedMetadata.push(originalMetadata);
|
|
4297
3901
|
}
|
|
4298
3902
|
}
|
|
4299
|
-
|
|
4300
|
-
chunks,
|
|
4301
|
-
chunk_ids,
|
|
3903
|
+
const output = {
|
|
4302
3904
|
metadata: enrichedMetadata,
|
|
4303
|
-
|
|
4304
|
-
count: chunks.length
|
|
3905
|
+
count: enrichedMetadata.length
|
|
4305
3906
|
};
|
|
3907
|
+
if (chunks !== undefined)
|
|
3908
|
+
output.chunks = chunks;
|
|
3909
|
+
if (chunk_ids !== undefined)
|
|
3910
|
+
output.chunk_ids = chunk_ids;
|
|
3911
|
+
if (scores !== undefined)
|
|
3912
|
+
output.scores = scores;
|
|
3913
|
+
return output;
|
|
4306
3914
|
}
|
|
4307
3915
|
}
|
|
4308
3916
|
var hierarchyJoin = (input, config) => {
|
|
4309
3917
|
return new HierarchyJoinTask(config).run(input);
|
|
4310
3918
|
};
|
|
4311
|
-
|
|
3919
|
+
Workflow17.prototype.hierarchyJoin = CreateWorkflow17(HierarchyJoinTask);
|
|
4312
3920
|
|
|
4313
3921
|
// src/task/KbToDocumentsTask.ts
|
|
4314
|
-
import { TypeKnowledgeBase as
|
|
4315
|
-
import { CreateWorkflow as
|
|
4316
|
-
var
|
|
3922
|
+
import { TypeKnowledgeBase as TypeKnowledgeBase5 } from "@workglow/knowledge-base";
|
|
3923
|
+
import { CreateWorkflow as CreateWorkflow18, Task as Task9, Workflow as Workflow18 } from "@workglow/task-graph";
|
|
3924
|
+
var inputSchema8 = {
|
|
4317
3925
|
type: "object",
|
|
4318
3926
|
properties: {
|
|
4319
|
-
knowledgeBase:
|
|
3927
|
+
knowledgeBase: TypeKnowledgeBase5({
|
|
4320
3928
|
title: "Knowledge Base",
|
|
4321
3929
|
description: "The knowledge base instance to list documents from"
|
|
4322
3930
|
}),
|
|
@@ -4330,7 +3938,7 @@ var inputSchema11 = {
|
|
|
4330
3938
|
required: ["knowledgeBase"],
|
|
4331
3939
|
additionalProperties: false
|
|
4332
3940
|
};
|
|
4333
|
-
var
|
|
3941
|
+
var outputSchema8 = {
|
|
4334
3942
|
type: "object",
|
|
4335
3943
|
properties: {
|
|
4336
3944
|
doc_id: {
|
|
@@ -4356,17 +3964,17 @@ var outputSchema11 = {
|
|
|
4356
3964
|
additionalProperties: false
|
|
4357
3965
|
};
|
|
4358
3966
|
|
|
4359
|
-
class KbToDocumentsTask extends
|
|
3967
|
+
class KbToDocumentsTask extends Task9 {
|
|
4360
3968
|
static type = "KbToDocumentsTask";
|
|
4361
3969
|
static category = "Vector Store";
|
|
4362
3970
|
static title = "Knowledge Base to Documents";
|
|
4363
3971
|
static description = "List documents from a knowledge base, optionally filtering to only those that need embedding";
|
|
4364
3972
|
static cacheable = false;
|
|
4365
3973
|
static inputSchema() {
|
|
4366
|
-
return
|
|
3974
|
+
return inputSchema8;
|
|
4367
3975
|
}
|
|
4368
3976
|
static outputSchema() {
|
|
4369
|
-
return
|
|
3977
|
+
return outputSchema8;
|
|
4370
3978
|
}
|
|
4371
3979
|
async execute(input, context) {
|
|
4372
3980
|
const { knowledgeBase, onlyStale = true } = input;
|
|
@@ -4397,10 +4005,10 @@ class KbToDocumentsTask extends Task12 {
|
|
|
4397
4005
|
var kbToDocuments = (input, config) => {
|
|
4398
4006
|
return new KbToDocumentsTask(config).run(input);
|
|
4399
4007
|
};
|
|
4400
|
-
|
|
4008
|
+
Workflow18.prototype.kbToDocuments = CreateWorkflow18(KbToDocumentsTask);
|
|
4401
4009
|
|
|
4402
4010
|
// src/task/ImageClassificationTask.ts
|
|
4403
|
-
import { CreateWorkflow as
|
|
4011
|
+
import { CreateWorkflow as CreateWorkflow19, Workflow as Workflow19 } from "@workglow/task-graph";
|
|
4404
4012
|
var modelSchema14 = TypeModel("model:ImageClassificationTask");
|
|
4405
4013
|
var ImageClassificationInputSchema = {
|
|
4406
4014
|
type: "object",
|
|
@@ -4460,12 +4068,12 @@ class ImageClassificationTask extends AiVisionTask {
|
|
|
4460
4068
|
var imageClassification = (input, config) => {
|
|
4461
4069
|
return new ImageClassificationTask(config).run(input);
|
|
4462
4070
|
};
|
|
4463
|
-
|
|
4071
|
+
Workflow19.prototype.imageClassification = CreateWorkflow19(ImageClassificationTask);
|
|
4464
4072
|
|
|
4465
4073
|
// src/task/ImageEmbeddingTask.ts
|
|
4466
|
-
import { CreateWorkflow as
|
|
4074
|
+
import { CreateWorkflow as CreateWorkflow20, Workflow as Workflow20 } from "@workglow/task-graph";
|
|
4467
4075
|
import {
|
|
4468
|
-
TypedArraySchema as
|
|
4076
|
+
TypedArraySchema as TypedArraySchema4
|
|
4469
4077
|
} from "@workglow/util/schema";
|
|
4470
4078
|
var modelSchema15 = TypeModel("model:ImageEmbeddingTask");
|
|
4471
4079
|
var ImageEmbeddingInputSchema = {
|
|
@@ -4480,7 +4088,7 @@ var ImageEmbeddingInputSchema = {
|
|
|
4480
4088
|
var ImageEmbeddingOutputSchema = {
|
|
4481
4089
|
type: "object",
|
|
4482
4090
|
properties: {
|
|
4483
|
-
vector: TypeSingleOrArray(
|
|
4091
|
+
vector: TypeSingleOrArray(TypedArraySchema4({
|
|
4484
4092
|
title: "Embedding",
|
|
4485
4093
|
description: "The image embedding vector"
|
|
4486
4094
|
}))
|
|
@@ -4504,10 +4112,10 @@ class ImageEmbeddingTask extends AiVisionTask {
|
|
|
4504
4112
|
var imageEmbedding = (input, config) => {
|
|
4505
4113
|
return new ImageEmbeddingTask(config).run(input);
|
|
4506
4114
|
};
|
|
4507
|
-
|
|
4115
|
+
Workflow20.prototype.imageEmbedding = CreateWorkflow20(ImageEmbeddingTask);
|
|
4508
4116
|
|
|
4509
4117
|
// src/task/ImageSegmentationTask.ts
|
|
4510
|
-
import { CreateWorkflow as
|
|
4118
|
+
import { CreateWorkflow as CreateWorkflow21, Workflow as Workflow21 } from "@workglow/task-graph";
|
|
4511
4119
|
var modelSchema16 = TypeModel("model:ImageSegmentationTask");
|
|
4512
4120
|
var ImageSegmentationInputSchema = {
|
|
4513
4121
|
type: "object",
|
|
@@ -4592,10 +4200,10 @@ class ImageSegmentationTask extends AiVisionTask {
|
|
|
4592
4200
|
var imageSegmentation = (input, config) => {
|
|
4593
4201
|
return new ImageSegmentationTask(config).run(input);
|
|
4594
4202
|
};
|
|
4595
|
-
|
|
4203
|
+
Workflow21.prototype.imageSegmentation = CreateWorkflow21(ImageSegmentationTask);
|
|
4596
4204
|
|
|
4597
4205
|
// src/task/ImageToTextTask.ts
|
|
4598
|
-
import { CreateWorkflow as
|
|
4206
|
+
import { CreateWorkflow as CreateWorkflow22, Workflow as Workflow22 } from "@workglow/task-graph";
|
|
4599
4207
|
var modelSchema17 = TypeModel("model:ImageToTextTask");
|
|
4600
4208
|
var generatedTextSchema = {
|
|
4601
4209
|
type: "string",
|
|
@@ -4647,10 +4255,10 @@ class ImageToTextTask extends AiVisionTask {
|
|
|
4647
4255
|
var imageToText = (input, config) => {
|
|
4648
4256
|
return new ImageToTextTask(config).run(input);
|
|
4649
4257
|
};
|
|
4650
|
-
|
|
4258
|
+
Workflow22.prototype.imageToText = CreateWorkflow22(ImageToTextTask);
|
|
4651
4259
|
|
|
4652
4260
|
// src/task/ModelInfoTask.ts
|
|
4653
|
-
import { CreateWorkflow as
|
|
4261
|
+
import { CreateWorkflow as CreateWorkflow23, Workflow as Workflow23 } from "@workglow/task-graph";
|
|
4654
4262
|
var modelSchema18 = TypeModel("model");
|
|
4655
4263
|
var ModelInfoInputSchema = {
|
|
4656
4264
|
type: "object",
|
|
@@ -4726,10 +4334,10 @@ class ModelInfoTask extends AiTask {
|
|
|
4726
4334
|
var modelInfo = (input, config) => {
|
|
4727
4335
|
return new ModelInfoTask(config).run(input);
|
|
4728
4336
|
};
|
|
4729
|
-
|
|
4337
|
+
Workflow23.prototype.modelInfo = CreateWorkflow23(ModelInfoTask);
|
|
4730
4338
|
|
|
4731
4339
|
// src/task/ModelSearchTask.ts
|
|
4732
|
-
import { CreateWorkflow as
|
|
4340
|
+
import { CreateWorkflow as CreateWorkflow24, Task as Task10, Workflow as Workflow24 } from "@workglow/task-graph";
|
|
4733
4341
|
var ModelSearchInputSchema = {
|
|
4734
4342
|
type: "object",
|
|
4735
4343
|
properties: {
|
|
@@ -4794,7 +4402,7 @@ var ModelSearchOutputSchema = {
|
|
|
4794
4402
|
additionalProperties: false
|
|
4795
4403
|
};
|
|
4796
4404
|
|
|
4797
|
-
class ModelSearchTask extends
|
|
4405
|
+
class ModelSearchTask extends Task10 {
|
|
4798
4406
|
static type = "ModelSearchTask";
|
|
4799
4407
|
static category = "AI Model";
|
|
4800
4408
|
static title = "Model Search";
|
|
@@ -4820,10 +4428,10 @@ class ModelSearchTask extends Task13 {
|
|
|
4820
4428
|
var modelSearch = (input, config) => {
|
|
4821
4429
|
return new ModelSearchTask(config).run(input);
|
|
4822
4430
|
};
|
|
4823
|
-
|
|
4431
|
+
Workflow24.prototype.modelSearch = CreateWorkflow24(ModelSearchTask);
|
|
4824
4432
|
|
|
4825
4433
|
// src/task/ObjectDetectionTask.ts
|
|
4826
|
-
import { CreateWorkflow as
|
|
4434
|
+
import { CreateWorkflow as CreateWorkflow25, Workflow as Workflow25 } from "@workglow/task-graph";
|
|
4827
4435
|
var modelSchema19 = TypeModel("model:ObjectDetectionTask");
|
|
4828
4436
|
var detectionSchema = {
|
|
4829
4437
|
type: "object",
|
|
@@ -4903,10 +4511,10 @@ class ObjectDetectionTask extends AiVisionTask {
|
|
|
4903
4511
|
var objectDetection = (input, config) => {
|
|
4904
4512
|
return new ObjectDetectionTask(config).run(input);
|
|
4905
4513
|
};
|
|
4906
|
-
|
|
4514
|
+
Workflow25.prototype.objectDetection = CreateWorkflow25(ObjectDetectionTask);
|
|
4907
4515
|
|
|
4908
4516
|
// src/task/PoseLandmarkerTask.ts
|
|
4909
|
-
import { CreateWorkflow as
|
|
4517
|
+
import { CreateWorkflow as CreateWorkflow26, Workflow as Workflow26 } from "@workglow/task-graph";
|
|
4910
4518
|
var modelSchema20 = TypeModel("model:PoseLandmarkerTask");
|
|
4911
4519
|
var TypePoseLandmark = {
|
|
4912
4520
|
type: "object",
|
|
@@ -5065,17 +4673,15 @@ class PoseLandmarkerTask extends AiVisionTask {
|
|
|
5065
4673
|
var poseLandmarker = (input, config) => {
|
|
5066
4674
|
return new PoseLandmarkerTask(config).run(input);
|
|
5067
4675
|
};
|
|
5068
|
-
|
|
4676
|
+
Workflow26.prototype.poseLandmarker = CreateWorkflow26(PoseLandmarkerTask);
|
|
5069
4677
|
|
|
5070
4678
|
// src/task/QueryExpanderTask.ts
|
|
5071
|
-
import { CreateWorkflow as
|
|
4679
|
+
import { CreateWorkflow as CreateWorkflow27, Task as Task11, Workflow as Workflow27 } from "@workglow/task-graph";
|
|
5072
4680
|
var QueryExpansionMethod = {
|
|
5073
4681
|
MULTI_QUERY: "multi-query",
|
|
5074
|
-
|
|
5075
|
-
SYNONYMS: "synonyms",
|
|
5076
|
-
PARAPHRASE: "paraphrase"
|
|
4682
|
+
SYNONYMS: "synonyms"
|
|
5077
4683
|
};
|
|
5078
|
-
var
|
|
4684
|
+
var inputSchema9 = {
|
|
5079
4685
|
type: "object",
|
|
5080
4686
|
properties: {
|
|
5081
4687
|
query: {
|
|
@@ -5097,17 +4703,12 @@ var inputSchema12 = {
|
|
|
5097
4703
|
minimum: 1,
|
|
5098
4704
|
maximum: 10,
|
|
5099
4705
|
default: 3
|
|
5100
|
-
},
|
|
5101
|
-
model: {
|
|
5102
|
-
type: "string",
|
|
5103
|
-
title: "Model",
|
|
5104
|
-
description: "LLM model to use for expansion (for HyDE and paraphrase methods)"
|
|
5105
4706
|
}
|
|
5106
4707
|
},
|
|
5107
4708
|
required: ["query"],
|
|
5108
4709
|
additionalProperties: false
|
|
5109
4710
|
};
|
|
5110
|
-
var
|
|
4711
|
+
var outputSchema9 = {
|
|
5111
4712
|
type: "object",
|
|
5112
4713
|
properties: {
|
|
5113
4714
|
query: {
|
|
@@ -5136,31 +4737,25 @@ var outputSchema12 = {
|
|
|
5136
4737
|
additionalProperties: false
|
|
5137
4738
|
};
|
|
5138
4739
|
|
|
5139
|
-
class QueryExpanderTask extends
|
|
4740
|
+
class QueryExpanderTask extends Task11 {
|
|
5140
4741
|
static type = "QueryExpanderTask";
|
|
5141
4742
|
static category = "RAG";
|
|
5142
4743
|
static title = "Query Expander";
|
|
5143
4744
|
static description = "Expand queries to improve retrieval coverage";
|
|
5144
4745
|
static cacheable = true;
|
|
5145
4746
|
static inputSchema() {
|
|
5146
|
-
return
|
|
4747
|
+
return inputSchema9;
|
|
5147
4748
|
}
|
|
5148
4749
|
static outputSchema() {
|
|
5149
|
-
return
|
|
4750
|
+
return outputSchema9;
|
|
5150
4751
|
}
|
|
5151
4752
|
async execute(input, context) {
|
|
5152
4753
|
const { query, method = QueryExpansionMethod.MULTI_QUERY, numVariations = 3 } = input;
|
|
5153
4754
|
let queries;
|
|
5154
4755
|
switch (method) {
|
|
5155
|
-
case QueryExpansionMethod.HYDE:
|
|
5156
|
-
queries = this.hydeExpansion(query, numVariations);
|
|
5157
|
-
break;
|
|
5158
4756
|
case QueryExpansionMethod.SYNONYMS:
|
|
5159
4757
|
queries = this.synonymExpansion(query, numVariations);
|
|
5160
4758
|
break;
|
|
5161
|
-
case QueryExpansionMethod.PARAPHRASE:
|
|
5162
|
-
queries = this.paraphraseExpansion(query, numVariations);
|
|
5163
|
-
break;
|
|
5164
4759
|
case QueryExpansionMethod.MULTI_QUERY:
|
|
5165
4760
|
default:
|
|
5166
4761
|
queries = this.multiQueryExpansion(query, numVariations);
|
|
@@ -5198,174 +4793,64 @@ class QueryExpanderTask extends Task14 {
|
|
|
5198
4793
|
if (!query.toLowerCase().startsWith("explain")) {
|
|
5199
4794
|
variations.push(`Explain ${query.toLowerCase()}`);
|
|
5200
4795
|
}
|
|
5201
|
-
for (let i = 0;i < Math.min(numVariations - 1, variations.length); i++) {
|
|
5202
|
-
if (variations[i] && !queries.includes(variations[i])) {
|
|
5203
|
-
queries.push(variations[i]);
|
|
5204
|
-
}
|
|
5205
|
-
}
|
|
5206
|
-
return queries;
|
|
5207
|
-
}
|
|
5208
|
-
|
|
5209
|
-
const queries = [query];
|
|
5210
|
-
const
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
|
|
5220
|
-
|
|
5221
|
-
|
|
5222
|
-
const
|
|
5223
|
-
|
|
5224
|
-
|
|
5225
|
-
|
|
5226
|
-
|
|
5227
|
-
|
|
5228
|
-
|
|
5229
|
-
|
|
5230
|
-
|
|
5231
|
-
|
|
5232
|
-
|
|
5233
|
-
|
|
5234
|
-
|
|
5235
|
-
|
|
5236
|
-
|
|
5237
|
-
|
|
5238
|
-
if (variationsGenerated >= numVariations - 1)
|
|
5239
|
-
break;
|
|
5240
|
-
const wordIndex = words.indexOf(word);
|
|
5241
|
-
if (wordIndex !== -1) {
|
|
5242
|
-
for (const syn of syns) {
|
|
5243
|
-
if (variationsGenerated >= numVariations - 1)
|
|
5244
|
-
break;
|
|
5245
|
-
const newWords = [...words];
|
|
5246
|
-
newWords[wordIndex] = syn;
|
|
5247
|
-
const newQuery = newWords.join(" ");
|
|
5248
|
-
const capitalizedQuery = this.preserveCapitalization(query, newQuery);
|
|
5249
|
-
if (!queries.includes(capitalizedQuery)) {
|
|
5250
|
-
queries.push(capitalizedQuery);
|
|
5251
|
-
variationsGenerated++;
|
|
5252
|
-
}
|
|
5253
|
-
}
|
|
5254
|
-
}
|
|
5255
|
-
}
|
|
5256
|
-
return queries;
|
|
5257
|
-
}
|
|
5258
|
-
paraphraseExpansion(query, numVariations) {
|
|
5259
|
-
const queries = [query];
|
|
5260
|
-
const paraphrases = [];
|
|
5261
|
-
paraphrases.push(`I need information about ${query.toLowerCase()}`);
|
|
5262
|
-
paraphrases.push(`Can you help me understand ${query.toLowerCase()}`);
|
|
5263
|
-
paraphrases.push(`I'm looking for details on ${query.toLowerCase()}`);
|
|
5264
|
-
for (let i = 0;i < Math.min(numVariations - 1, paraphrases.length); i++) {
|
|
5265
|
-
if (!queries.includes(paraphrases[i])) {
|
|
5266
|
-
queries.push(paraphrases[i]);
|
|
5267
|
-
}
|
|
5268
|
-
}
|
|
5269
|
-
return queries;
|
|
5270
|
-
}
|
|
5271
|
-
preserveCapitalization(original, modified) {
|
|
5272
|
-
if (original[0] === original[0].toUpperCase()) {
|
|
5273
|
-
return modified.charAt(0).toUpperCase() + modified.slice(1);
|
|
5274
|
-
}
|
|
5275
|
-
return modified;
|
|
5276
|
-
}
|
|
5277
|
-
}
|
|
5278
|
-
var queryExpander = (input, config) => {
|
|
5279
|
-
return new QueryExpanderTask(config).run(input);
|
|
5280
|
-
};
|
|
5281
|
-
Workflow30.prototype.queryExpander = CreateWorkflow30(QueryExpanderTask);
|
|
5282
|
-
|
|
5283
|
-
// src/task/RerankerTask.ts
|
|
5284
|
-
import { CreateWorkflow as CreateWorkflow32, Task as Task15, Workflow as Workflow32 } from "@workglow/task-graph";
|
|
5285
|
-
|
|
5286
|
-
// src/task/TextClassificationTask.ts
|
|
5287
|
-
import { CreateWorkflow as CreateWorkflow31, Workflow as Workflow31 } from "@workglow/task-graph";
|
|
5288
|
-
var modelSchema21 = TypeModel("model:TextClassificationTask");
|
|
5289
|
-
var TextClassificationInputSchema = {
|
|
5290
|
-
type: "object",
|
|
5291
|
-
properties: {
|
|
5292
|
-
text: {
|
|
5293
|
-
type: "string",
|
|
5294
|
-
title: "Text",
|
|
5295
|
-
description: "The text to classify"
|
|
5296
|
-
},
|
|
5297
|
-
candidateLabels: {
|
|
5298
|
-
type: "array",
|
|
5299
|
-
items: {
|
|
5300
|
-
type: "string"
|
|
5301
|
-
},
|
|
5302
|
-
title: "Candidate Labels",
|
|
5303
|
-
description: "List of candidate labels (optional, if provided uses zero-shot classification)",
|
|
5304
|
-
"x-ui-group": "Configuration"
|
|
5305
|
-
},
|
|
5306
|
-
maxCategories: {
|
|
5307
|
-
type: "number",
|
|
5308
|
-
minimum: 1,
|
|
5309
|
-
maximum: 1000,
|
|
5310
|
-
default: 5,
|
|
5311
|
-
title: "Max Categories",
|
|
5312
|
-
description: "The maximum number of categories to return",
|
|
5313
|
-
"x-ui-group": "Configuration"
|
|
5314
|
-
},
|
|
5315
|
-
model: modelSchema21
|
|
5316
|
-
},
|
|
5317
|
-
required: ["text", "model"],
|
|
5318
|
-
additionalProperties: false
|
|
5319
|
-
};
|
|
5320
|
-
var TextClassificationOutputSchema = {
|
|
5321
|
-
type: "object",
|
|
5322
|
-
properties: {
|
|
5323
|
-
categories: {
|
|
5324
|
-
type: "array",
|
|
5325
|
-
items: {
|
|
5326
|
-
type: "object",
|
|
5327
|
-
properties: {
|
|
5328
|
-
label: {
|
|
5329
|
-
type: "string",
|
|
5330
|
-
title: "Label",
|
|
5331
|
-
description: "The name of the category"
|
|
5332
|
-
},
|
|
5333
|
-
score: {
|
|
5334
|
-
type: "number",
|
|
5335
|
-
title: "Score",
|
|
5336
|
-
description: "The confidence score for this category"
|
|
4796
|
+
for (let i = 0;i < Math.min(numVariations - 1, variations.length); i++) {
|
|
4797
|
+
if (variations[i] && !queries.includes(variations[i])) {
|
|
4798
|
+
queries.push(variations[i]);
|
|
4799
|
+
}
|
|
4800
|
+
}
|
|
4801
|
+
return queries;
|
|
4802
|
+
}
|
|
4803
|
+
synonymExpansion(query, numVariations) {
|
|
4804
|
+
const queries = [query];
|
|
4805
|
+
const synonyms = {
|
|
4806
|
+
find: ["locate", "discover", "search for"],
|
|
4807
|
+
create: ["make", "build", "generate"],
|
|
4808
|
+
delete: ["remove", "erase", "eliminate"],
|
|
4809
|
+
update: ["modify", "change", "edit"],
|
|
4810
|
+
show: ["display", "present", "reveal"],
|
|
4811
|
+
explain: ["describe", "clarify", "elaborate"],
|
|
4812
|
+
help: ["assist", "aid", "support"],
|
|
4813
|
+
problem: ["issue", "challenge", "difficulty"],
|
|
4814
|
+
solution: ["answer", "resolution", "fix"],
|
|
4815
|
+
method: ["approach", "technique", "way"]
|
|
4816
|
+
};
|
|
4817
|
+
const words = query.toLowerCase().split(/\s+/);
|
|
4818
|
+
let variationsGenerated = 0;
|
|
4819
|
+
for (const [word, syns] of Object.entries(synonyms)) {
|
|
4820
|
+
if (variationsGenerated >= numVariations - 1)
|
|
4821
|
+
break;
|
|
4822
|
+
const wordIndex = words.indexOf(word);
|
|
4823
|
+
if (wordIndex !== -1) {
|
|
4824
|
+
for (const syn of syns) {
|
|
4825
|
+
if (variationsGenerated >= numVariations - 1)
|
|
4826
|
+
break;
|
|
4827
|
+
const newWords = [...words];
|
|
4828
|
+
newWords[wordIndex] = syn;
|
|
4829
|
+
const capitalizedQuery = this.preserveCapitalization(query, newWords.join(" "));
|
|
4830
|
+
if (!queries.includes(capitalizedQuery)) {
|
|
4831
|
+
queries.push(capitalizedQuery);
|
|
4832
|
+
variationsGenerated++;
|
|
5337
4833
|
}
|
|
5338
|
-
}
|
|
5339
|
-
|
|
5340
|
-
additionalProperties: false
|
|
5341
|
-
},
|
|
5342
|
-
title: "Categories",
|
|
5343
|
-
description: "The classification categories with their scores"
|
|
4834
|
+
}
|
|
4835
|
+
}
|
|
5344
4836
|
}
|
|
5345
|
-
|
|
5346
|
-
required: ["categories"],
|
|
5347
|
-
additionalProperties: false
|
|
5348
|
-
};
|
|
5349
|
-
|
|
5350
|
-
class TextClassificationTask extends AiTask {
|
|
5351
|
-
static type = "TextClassificationTask";
|
|
5352
|
-
static category = "AI Text Model";
|
|
5353
|
-
static title = "Text Classifier";
|
|
5354
|
-
static description = "Classifies text into categories using language models. Supports zero-shot classification when candidate labels are provided.";
|
|
5355
|
-
static inputSchema() {
|
|
5356
|
-
return TextClassificationInputSchema;
|
|
4837
|
+
return queries;
|
|
5357
4838
|
}
|
|
5358
|
-
|
|
5359
|
-
|
|
4839
|
+
preserveCapitalization(original, modified) {
|
|
4840
|
+
if (original[0] === original[0].toUpperCase()) {
|
|
4841
|
+
return modified.charAt(0).toUpperCase() + modified.slice(1);
|
|
4842
|
+
}
|
|
4843
|
+
return modified;
|
|
5360
4844
|
}
|
|
5361
4845
|
}
|
|
5362
|
-
var
|
|
5363
|
-
return new
|
|
4846
|
+
var queryExpander = (input, config) => {
|
|
4847
|
+
return new QueryExpanderTask(config).run(input);
|
|
5364
4848
|
};
|
|
5365
|
-
|
|
4849
|
+
Workflow27.prototype.queryExpander = CreateWorkflow27(QueryExpanderTask);
|
|
5366
4850
|
|
|
5367
4851
|
// src/task/RerankerTask.ts
|
|
5368
|
-
|
|
4852
|
+
import { CreateWorkflow as CreateWorkflow28, Task as Task12, Workflow as Workflow28 } from "@workglow/task-graph";
|
|
4853
|
+
var inputSchema10 = {
|
|
5369
4854
|
type: "object",
|
|
5370
4855
|
properties: {
|
|
5371
4856
|
query: {
|
|
@@ -5403,20 +4888,16 @@ var inputSchema13 = {
|
|
|
5403
4888
|
},
|
|
5404
4889
|
method: {
|
|
5405
4890
|
type: "string",
|
|
5406
|
-
enum: ["
|
|
4891
|
+
enum: ["reciprocal-rank-fusion", "simple"],
|
|
5407
4892
|
title: "Reranking Method",
|
|
5408
4893
|
description: "Method to use for reranking",
|
|
5409
4894
|
default: "simple"
|
|
5410
|
-
}
|
|
5411
|
-
model: TypeModel("model:RerankerTask", {
|
|
5412
|
-
title: "Reranker Model",
|
|
5413
|
-
description: "Cross-encoder model to use for reranking"
|
|
5414
|
-
})
|
|
4895
|
+
}
|
|
5415
4896
|
},
|
|
5416
4897
|
required: ["query", "chunks"],
|
|
5417
4898
|
additionalProperties: false
|
|
5418
4899
|
};
|
|
5419
|
-
var
|
|
4900
|
+
var outputSchema10 = {
|
|
5420
4901
|
type: "object",
|
|
5421
4902
|
properties: {
|
|
5422
4903
|
chunks: {
|
|
@@ -5457,28 +4938,22 @@ var outputSchema13 = {
|
|
|
5457
4938
|
additionalProperties: false
|
|
5458
4939
|
};
|
|
5459
4940
|
|
|
5460
|
-
class RerankerTask extends
|
|
4941
|
+
class RerankerTask extends Task12 {
|
|
5461
4942
|
static type = "RerankerTask";
|
|
5462
4943
|
static category = "RAG";
|
|
5463
4944
|
static title = "Reranker";
|
|
5464
4945
|
static description = "Rerank retrieved chunks to improve relevance";
|
|
5465
4946
|
static cacheable = true;
|
|
5466
4947
|
static inputSchema() {
|
|
5467
|
-
return
|
|
4948
|
+
return inputSchema10;
|
|
5468
4949
|
}
|
|
5469
4950
|
static outputSchema() {
|
|
5470
|
-
return
|
|
4951
|
+
return outputSchema10;
|
|
5471
4952
|
}
|
|
5472
4953
|
async execute(input, context) {
|
|
5473
|
-
const { query, chunks, scores = [], metadata = [], topK, method = "simple"
|
|
4954
|
+
const { query, chunks, scores = [], metadata = [], topK, method = "simple" } = input;
|
|
5474
4955
|
let rankedItems;
|
|
5475
4956
|
switch (method) {
|
|
5476
|
-
case "cross-encoder":
|
|
5477
|
-
if (!model) {
|
|
5478
|
-
throw new Error("No cross-encoder model found. Please provide a model with format 'model:RerankerTask'.");
|
|
5479
|
-
}
|
|
5480
|
-
rankedItems = await this.crossEncoderRerank(query, chunks, scores, metadata, model, context);
|
|
5481
|
-
break;
|
|
5482
4957
|
case "reciprocal-rank-fusion":
|
|
5483
4958
|
rankedItems = this.reciprocalRankFusion(chunks, scores, metadata);
|
|
5484
4959
|
break;
|
|
@@ -5498,44 +4973,6 @@ class RerankerTask extends Task15 {
|
|
|
5498
4973
|
count: rankedItems.length
|
|
5499
4974
|
};
|
|
5500
4975
|
}
|
|
5501
|
-
async crossEncoderRerank(query, chunks, scores, metadata, model, context) {
|
|
5502
|
-
if (chunks.length === 0) {
|
|
5503
|
-
return [];
|
|
5504
|
-
}
|
|
5505
|
-
if (!model) {
|
|
5506
|
-
throw new Error("No cross-encoder model found. Please provide a model or register a TextClassificationTask model.");
|
|
5507
|
-
}
|
|
5508
|
-
const items = await Promise.all(chunks.map(async (chunk, index) => {
|
|
5509
|
-
const pairText = `${query} [SEP] ${chunk}`;
|
|
5510
|
-
const task = context.own(new TextClassificationTask({ defaults: { model } }));
|
|
5511
|
-
const result = await task.run({ text: pairText, maxCategories: 2 });
|
|
5512
|
-
const crossScore = this.extractCrossEncoderScore(result.categories);
|
|
5513
|
-
return {
|
|
5514
|
-
chunk,
|
|
5515
|
-
score: Number.isFinite(crossScore) ? crossScore : scores[index] || 0,
|
|
5516
|
-
metadata: metadata[index],
|
|
5517
|
-
originalIndex: index
|
|
5518
|
-
};
|
|
5519
|
-
}));
|
|
5520
|
-
items.sort((a, b) => b.score - a.score);
|
|
5521
|
-
return items;
|
|
5522
|
-
}
|
|
5523
|
-
extractCrossEncoderScore(categories) {
|
|
5524
|
-
if (!categories || categories.length === 0) {
|
|
5525
|
-
return 0;
|
|
5526
|
-
}
|
|
5527
|
-
const preferred = categories.find((category) => /^(label_1|positive|relevant|yes|true)$/i.test(category.label));
|
|
5528
|
-
if (preferred) {
|
|
5529
|
-
return preferred.score;
|
|
5530
|
-
}
|
|
5531
|
-
let best = categories[0].score;
|
|
5532
|
-
for (let i = 1;i < categories.length; i++) {
|
|
5533
|
-
if (categories[i].score > best) {
|
|
5534
|
-
best = categories[i].score;
|
|
5535
|
-
}
|
|
5536
|
-
}
|
|
5537
|
-
return best;
|
|
5538
|
-
}
|
|
5539
4976
|
simpleRerank(query, chunks, scores, metadata) {
|
|
5540
4977
|
const queryLower = query.toLowerCase();
|
|
5541
4978
|
const queryWords = queryLower.split(/\s+/).filter((w) => w.length > 0);
|
|
@@ -5543,7 +4980,6 @@ class RerankerTask extends Task15 {
|
|
|
5543
4980
|
const chunkLower = chunk.toLowerCase();
|
|
5544
4981
|
const initialScore = scores[index] || 0;
|
|
5545
4982
|
let keywordScore = 0;
|
|
5546
|
-
let exactMatchBonus = 0;
|
|
5547
4983
|
for (const word of queryWords) {
|
|
5548
4984
|
const regex = new RegExp(word, "gi");
|
|
5549
4985
|
const matches = chunkLower.match(regex);
|
|
@@ -5551,33 +4987,23 @@ class RerankerTask extends Task15 {
|
|
|
5551
4987
|
keywordScore += matches.length;
|
|
5552
4988
|
}
|
|
5553
4989
|
}
|
|
5554
|
-
|
|
5555
|
-
exactMatchBonus = 0.5;
|
|
5556
|
-
}
|
|
4990
|
+
const exactMatchBonus = chunkLower.includes(queryLower) ? 0.5 : 0;
|
|
5557
4991
|
const normalizedKeywordScore = Math.min(keywordScore / (queryWords.length * 3), 1);
|
|
5558
4992
|
const positionPenalty = Math.log(index + 1) / 10;
|
|
5559
4993
|
const combinedScore = initialScore * 0.4 + normalizedKeywordScore * 0.4 + exactMatchBonus * 0.2 - positionPenalty;
|
|
5560
|
-
return {
|
|
5561
|
-
chunk,
|
|
5562
|
-
score: combinedScore,
|
|
5563
|
-
metadata: metadata[index],
|
|
5564
|
-
originalIndex: index
|
|
5565
|
-
};
|
|
4994
|
+
return { chunk, score: combinedScore, metadata: metadata[index], originalIndex: index };
|
|
5566
4995
|
});
|
|
5567
4996
|
items.sort((a, b) => b.score - a.score);
|
|
5568
4997
|
return items;
|
|
5569
4998
|
}
|
|
5570
4999
|
reciprocalRankFusion(chunks, scores, metadata) {
|
|
5571
5000
|
const k = 60;
|
|
5572
|
-
const items = chunks.map((chunk, index) => {
|
|
5573
|
-
|
|
5574
|
-
|
|
5575
|
-
|
|
5576
|
-
|
|
5577
|
-
|
|
5578
|
-
originalIndex: index
|
|
5579
|
-
};
|
|
5580
|
-
});
|
|
5001
|
+
const items = chunks.map((chunk, index) => ({
|
|
5002
|
+
chunk,
|
|
5003
|
+
score: 1 / (k + index + 1),
|
|
5004
|
+
metadata: metadata[index],
|
|
5005
|
+
originalIndex: index
|
|
5006
|
+
}));
|
|
5581
5007
|
items.sort((a, b) => b.score - a.score);
|
|
5582
5008
|
return items;
|
|
5583
5009
|
}
|
|
@@ -5585,13 +5011,13 @@ class RerankerTask extends Task15 {
|
|
|
5585
5011
|
var reranker = (input, config) => {
|
|
5586
5012
|
return new RerankerTask(config).run(input);
|
|
5587
5013
|
};
|
|
5588
|
-
|
|
5014
|
+
Workflow28.prototype.reranker = CreateWorkflow28(RerankerTask);
|
|
5589
5015
|
|
|
5590
5016
|
// src/task/StructuralParserTask.ts
|
|
5591
5017
|
import { StructuralParser } from "@workglow/knowledge-base";
|
|
5592
|
-
import { CreateWorkflow as
|
|
5018
|
+
import { CreateWorkflow as CreateWorkflow29, Task as Task13, Workflow as Workflow29 } from "@workglow/task-graph";
|
|
5593
5019
|
import { uuid4 as uuid42 } from "@workglow/util";
|
|
5594
|
-
var
|
|
5020
|
+
var inputSchema11 = {
|
|
5595
5021
|
type: "object",
|
|
5596
5022
|
properties: {
|
|
5597
5023
|
text: {
|
|
@@ -5625,7 +5051,7 @@ var inputSchema14 = {
|
|
|
5625
5051
|
required: ["text", "title"],
|
|
5626
5052
|
additionalProperties: false
|
|
5627
5053
|
};
|
|
5628
|
-
var
|
|
5054
|
+
var outputSchema11 = {
|
|
5629
5055
|
type: "object",
|
|
5630
5056
|
properties: {
|
|
5631
5057
|
doc_id: {
|
|
@@ -5649,17 +5075,17 @@ var outputSchema14 = {
|
|
|
5649
5075
|
additionalProperties: false
|
|
5650
5076
|
};
|
|
5651
5077
|
|
|
5652
|
-
class StructuralParserTask extends
|
|
5078
|
+
class StructuralParserTask extends Task13 {
|
|
5653
5079
|
static type = "StructuralParserTask";
|
|
5654
5080
|
static category = "Document";
|
|
5655
5081
|
static title = "Structural Parser";
|
|
5656
5082
|
static description = "Parse documents into hierarchical tree structure";
|
|
5657
5083
|
static cacheable = true;
|
|
5658
5084
|
static inputSchema() {
|
|
5659
|
-
return
|
|
5085
|
+
return inputSchema11;
|
|
5660
5086
|
}
|
|
5661
5087
|
static outputSchema() {
|
|
5662
|
-
return
|
|
5088
|
+
return outputSchema11;
|
|
5663
5089
|
}
|
|
5664
5090
|
async execute(input, context) {
|
|
5665
5091
|
const { text, title, format = "auto", sourceUri, doc_id: providedDocId } = input;
|
|
@@ -5692,16 +5118,16 @@ class StructuralParserTask extends Task16 {
|
|
|
5692
5118
|
var structuralParser = (input, config) => {
|
|
5693
5119
|
return new StructuralParserTask(config).run(input);
|
|
5694
5120
|
};
|
|
5695
|
-
|
|
5121
|
+
Workflow29.prototype.structuralParser = CreateWorkflow29(StructuralParserTask);
|
|
5696
5122
|
|
|
5697
5123
|
// src/task/StructuredGenerationTask.ts
|
|
5698
|
-
import { CreateWorkflow as
|
|
5124
|
+
import { CreateWorkflow as CreateWorkflow30, TaskConfigurationError as TaskConfigurationError4, TaskError, Workflow as Workflow30 } from "@workglow/task-graph";
|
|
5699
5125
|
import { compileSchema as compileSchema2 } from "@workglow/util/schema";
|
|
5700
|
-
var
|
|
5126
|
+
var modelSchema21 = TypeModel("model:StructuredGenerationTask");
|
|
5701
5127
|
var StructuredGenerationInputSchema = {
|
|
5702
5128
|
type: "object",
|
|
5703
5129
|
properties: {
|
|
5704
|
-
model:
|
|
5130
|
+
model: modelSchema21,
|
|
5705
5131
|
prompt: {
|
|
5706
5132
|
type: "string",
|
|
5707
5133
|
title: "Prompt",
|
|
@@ -5869,17 +5295,18 @@ class StructuredGenerationTask extends StreamingAiTask {
|
|
|
5869
5295
|
var structuredGeneration = (input, config) => {
|
|
5870
5296
|
return new StructuredGenerationTask(config).run(input);
|
|
5871
5297
|
};
|
|
5872
|
-
|
|
5298
|
+
Workflow30.prototype.structuredGeneration = CreateWorkflow30(StructuredGenerationTask);
|
|
5873
5299
|
|
|
5874
5300
|
// src/task/TextChunkerTask.ts
|
|
5875
|
-
import {
|
|
5301
|
+
import { ChunkRecordArraySchema as ChunkRecordArraySchema3 } from "@workglow/knowledge-base";
|
|
5302
|
+
import { CreateWorkflow as CreateWorkflow31, Task as Task14, Workflow as Workflow31 } from "@workglow/task-graph";
|
|
5876
5303
|
var ChunkingStrategy = {
|
|
5877
5304
|
FIXED: "fixed",
|
|
5878
5305
|
SENTENCE: "sentence",
|
|
5879
5306
|
PARAGRAPH: "paragraph",
|
|
5880
5307
|
SEMANTIC: "semantic"
|
|
5881
5308
|
};
|
|
5882
|
-
var
|
|
5309
|
+
var inputSchema12 = {
|
|
5883
5310
|
type: "object",
|
|
5884
5311
|
properties: {
|
|
5885
5312
|
text: {
|
|
@@ -5887,6 +5314,11 @@ var inputSchema15 = {
|
|
|
5887
5314
|
title: "Text",
|
|
5888
5315
|
description: "The text to chunk"
|
|
5889
5316
|
},
|
|
5317
|
+
doc_id: {
|
|
5318
|
+
type: "string",
|
|
5319
|
+
title: "Document ID",
|
|
5320
|
+
description: "Optional document ID stamped onto each chunk. When omitted, chunks are emitted without a doc_id and the output also has no doc_id."
|
|
5321
|
+
},
|
|
5890
5322
|
chunkSize: {
|
|
5891
5323
|
type: "number",
|
|
5892
5324
|
title: "Chunk Size",
|
|
@@ -5912,88 +5344,99 @@ var inputSchema15 = {
|
|
|
5912
5344
|
required: ["text"],
|
|
5913
5345
|
additionalProperties: false
|
|
5914
5346
|
};
|
|
5915
|
-
var
|
|
5347
|
+
var outputSchema12 = {
|
|
5916
5348
|
type: "object",
|
|
5917
5349
|
properties: {
|
|
5918
|
-
|
|
5350
|
+
doc_id: {
|
|
5351
|
+
type: "string",
|
|
5352
|
+
title: "Document ID",
|
|
5353
|
+
description: "The document ID (only emitted when provided in input)"
|
|
5354
|
+
},
|
|
5355
|
+
chunks: ChunkRecordArraySchema3,
|
|
5356
|
+
text: {
|
|
5919
5357
|
type: "array",
|
|
5920
5358
|
items: { type: "string" },
|
|
5921
|
-
title: "
|
|
5922
|
-
description: "
|
|
5359
|
+
title: "Texts",
|
|
5360
|
+
description: "Chunk texts (for TextEmbeddingTask)"
|
|
5923
5361
|
},
|
|
5924
|
-
|
|
5925
|
-
type: "
|
|
5926
|
-
|
|
5927
|
-
|
|
5928
|
-
properties: {
|
|
5929
|
-
index: { type: "number" },
|
|
5930
|
-
startChar: { type: "number" },
|
|
5931
|
-
endChar: { type: "number" },
|
|
5932
|
-
length: { type: "number" }
|
|
5933
|
-
},
|
|
5934
|
-
additionalProperties: false
|
|
5935
|
-
},
|
|
5936
|
-
title: "Chunk Metadata",
|
|
5937
|
-
description: "Metadata for each chunk"
|
|
5362
|
+
count: {
|
|
5363
|
+
type: "number",
|
|
5364
|
+
title: "Count",
|
|
5365
|
+
description: "Number of chunks generated"
|
|
5938
5366
|
}
|
|
5939
5367
|
},
|
|
5940
|
-
required: ["chunks", "
|
|
5368
|
+
required: ["chunks", "text", "count"],
|
|
5941
5369
|
additionalProperties: false
|
|
5942
5370
|
};
|
|
5943
5371
|
|
|
5944
|
-
class TextChunkerTask extends
|
|
5372
|
+
class TextChunkerTask extends Task14 {
|
|
5945
5373
|
static type = "TextChunkerTask";
|
|
5946
5374
|
static category = "Document";
|
|
5947
5375
|
static title = "Text Chunker";
|
|
5948
5376
|
static description = "Splits text into chunks using various strategies (fixed, sentence, paragraph)";
|
|
5949
5377
|
static cacheable = true;
|
|
5950
5378
|
static inputSchema() {
|
|
5951
|
-
return
|
|
5379
|
+
return inputSchema12;
|
|
5952
5380
|
}
|
|
5953
5381
|
static outputSchema() {
|
|
5954
|
-
return
|
|
5382
|
+
return outputSchema12;
|
|
5955
5383
|
}
|
|
5956
5384
|
async execute(input, context) {
|
|
5957
|
-
const {
|
|
5958
|
-
|
|
5959
|
-
|
|
5385
|
+
const {
|
|
5386
|
+
text,
|
|
5387
|
+
doc_id,
|
|
5388
|
+
chunkSize = 512,
|
|
5389
|
+
chunkOverlap = 50,
|
|
5390
|
+
strategy = ChunkingStrategy.FIXED
|
|
5391
|
+
} = input;
|
|
5392
|
+
let rawChunks;
|
|
5960
5393
|
switch (strategy) {
|
|
5961
5394
|
case ChunkingStrategy.SENTENCE:
|
|
5962
|
-
|
|
5395
|
+
case ChunkingStrategy.SEMANTIC:
|
|
5396
|
+
rawChunks = this.chunkBySentence(text, chunkSize, chunkOverlap);
|
|
5963
5397
|
break;
|
|
5964
5398
|
case ChunkingStrategy.PARAGRAPH:
|
|
5965
|
-
|
|
5966
|
-
break;
|
|
5967
|
-
case ChunkingStrategy.SEMANTIC:
|
|
5968
|
-
({ chunks, metadata } = this.chunkBySentence(text, chunkSize, chunkOverlap));
|
|
5399
|
+
rawChunks = this.chunkByParagraph(text, chunkSize, chunkOverlap);
|
|
5969
5400
|
break;
|
|
5970
5401
|
case ChunkingStrategy.FIXED:
|
|
5971
5402
|
default:
|
|
5972
|
-
|
|
5403
|
+
rawChunks = this.chunkFixed(text, chunkSize, chunkOverlap);
|
|
5973
5404
|
break;
|
|
5974
5405
|
}
|
|
5975
|
-
|
|
5406
|
+
const nodePath = doc_id ? [doc_id] : [];
|
|
5407
|
+
const chunks = rawChunks.map((raw, index) => ({
|
|
5408
|
+
chunkId: doc_id ? `${doc_id}:${index}` : `chunk:${index}:${raw.startChar}`,
|
|
5409
|
+
doc_id: doc_id ?? "",
|
|
5410
|
+
text: raw.text,
|
|
5411
|
+
nodePath,
|
|
5412
|
+
depth: nodePath.length,
|
|
5413
|
+
...doc_id ? { leafNodeId: doc_id } : {},
|
|
5414
|
+
index,
|
|
5415
|
+
startChar: raw.startChar,
|
|
5416
|
+
endChar: raw.endChar
|
|
5417
|
+
}));
|
|
5418
|
+
const output = {
|
|
5419
|
+
chunks,
|
|
5420
|
+
text: chunks.map((c) => c.text),
|
|
5421
|
+
count: chunks.length
|
|
5422
|
+
};
|
|
5423
|
+
if (doc_id)
|
|
5424
|
+
output.doc_id = doc_id;
|
|
5425
|
+
return output;
|
|
5976
5426
|
}
|
|
5977
5427
|
chunkFixed(text, chunkSize, chunkOverlap) {
|
|
5978
5428
|
const chunks = [];
|
|
5979
|
-
const metadata = [];
|
|
5980
5429
|
let startChar = 0;
|
|
5981
|
-
let index = 0;
|
|
5982
5430
|
while (startChar < text.length) {
|
|
5983
5431
|
const endChar = Math.min(startChar + chunkSize, text.length);
|
|
5984
|
-
|
|
5985
|
-
|
|
5986
|
-
metadata.push({
|
|
5987
|
-
index,
|
|
5432
|
+
chunks.push({
|
|
5433
|
+
text: text.substring(startChar, endChar),
|
|
5988
5434
|
startChar,
|
|
5989
|
-
endChar
|
|
5990
|
-
length: chunk.length
|
|
5435
|
+
endChar
|
|
5991
5436
|
});
|
|
5992
|
-
|
|
5993
|
-
startChar += step;
|
|
5994
|
-
index++;
|
|
5437
|
+
startChar += Math.max(1, chunkSize - chunkOverlap);
|
|
5995
5438
|
}
|
|
5996
|
-
return
|
|
5439
|
+
return chunks;
|
|
5997
5440
|
}
|
|
5998
5441
|
chunkBySentence(text, chunkSize, chunkOverlap) {
|
|
5999
5442
|
const sentenceRegex = /[.!?]+[\s\n]+/g;
|
|
@@ -6002,8 +5445,7 @@ class TextChunkerTask extends Task17 {
|
|
|
6002
5445
|
let lastIndex = 0;
|
|
6003
5446
|
let match;
|
|
6004
5447
|
while ((match = sentenceRegex.exec(text)) !== null) {
|
|
6005
|
-
|
|
6006
|
-
sentences.push(sentence);
|
|
5448
|
+
sentences.push(text.substring(lastIndex, match.index + match[0].length));
|
|
6007
5449
|
sentenceStarts.push(lastIndex);
|
|
6008
5450
|
lastIndex = match.index + match[0].length;
|
|
6009
5451
|
}
|
|
@@ -6012,22 +5454,17 @@ class TextChunkerTask extends Task17 {
|
|
|
6012
5454
|
sentenceStarts.push(lastIndex);
|
|
6013
5455
|
}
|
|
6014
5456
|
const chunks = [];
|
|
6015
|
-
const metadata = [];
|
|
6016
5457
|
let currentChunk = "";
|
|
6017
5458
|
let currentStartChar = 0;
|
|
6018
|
-
let index = 0;
|
|
6019
5459
|
for (let i = 0;i < sentences.length; i++) {
|
|
6020
5460
|
const sentence = sentences[i];
|
|
6021
5461
|
const sentenceStart = sentenceStarts[i];
|
|
6022
5462
|
if (currentChunk.length > 0 && currentChunk.length + sentence.length > chunkSize) {
|
|
6023
|
-
chunks.push(
|
|
6024
|
-
|
|
6025
|
-
index,
|
|
5463
|
+
chunks.push({
|
|
5464
|
+
text: currentChunk.trim(),
|
|
6026
5465
|
startChar: currentStartChar,
|
|
6027
|
-
endChar: currentStartChar + currentChunk.length
|
|
6028
|
-
length: currentChunk.trim().length
|
|
5466
|
+
endChar: currentStartChar + currentChunk.length
|
|
6029
5467
|
});
|
|
6030
|
-
index++;
|
|
6031
5468
|
if (chunkOverlap > 0) {
|
|
6032
5469
|
let overlapText = "";
|
|
6033
5470
|
let j = i - 1;
|
|
@@ -6049,37 +5486,30 @@ class TextChunkerTask extends Task17 {
|
|
|
6049
5486
|
}
|
|
6050
5487
|
}
|
|
6051
5488
|
if (currentChunk.length > 0) {
|
|
6052
|
-
chunks.push(
|
|
6053
|
-
|
|
6054
|
-
index,
|
|
5489
|
+
chunks.push({
|
|
5490
|
+
text: currentChunk.trim(),
|
|
6055
5491
|
startChar: currentStartChar,
|
|
6056
|
-
endChar: currentStartChar + currentChunk.length
|
|
6057
|
-
length: currentChunk.trim().length
|
|
5492
|
+
endChar: currentStartChar + currentChunk.length
|
|
6058
5493
|
});
|
|
6059
5494
|
}
|
|
6060
|
-
return
|
|
5495
|
+
return chunks;
|
|
6061
5496
|
}
|
|
6062
5497
|
chunkByParagraph(text, chunkSize, chunkOverlap) {
|
|
6063
5498
|
const paragraphs = text.split(/\n\s*\n/).filter((p) => p.trim().length > 0);
|
|
6064
5499
|
const chunks = [];
|
|
6065
|
-
const metadata = [];
|
|
6066
5500
|
let currentChunk = "";
|
|
6067
5501
|
let currentStartChar = 0;
|
|
6068
|
-
let index = 0;
|
|
6069
5502
|
let charPosition = 0;
|
|
6070
5503
|
for (let i = 0;i < paragraphs.length; i++) {
|
|
6071
5504
|
const paragraph = paragraphs[i].trim();
|
|
6072
5505
|
const paragraphStart = text.indexOf(paragraph, charPosition);
|
|
6073
5506
|
charPosition = paragraphStart + paragraph.length;
|
|
6074
5507
|
if (currentChunk.length > 0 && currentChunk.length + paragraph.length + 2 > chunkSize) {
|
|
6075
|
-
chunks.push(
|
|
6076
|
-
|
|
6077
|
-
index,
|
|
5508
|
+
chunks.push({
|
|
5509
|
+
text: currentChunk.trim(),
|
|
6078
5510
|
startChar: currentStartChar,
|
|
6079
|
-
endChar: currentStartChar + currentChunk.length
|
|
6080
|
-
length: currentChunk.trim().length
|
|
5511
|
+
endChar: currentStartChar + currentChunk.length
|
|
6081
5512
|
});
|
|
6082
|
-
index++;
|
|
6083
5513
|
if (chunkOverlap > 0 && i > 0) {
|
|
6084
5514
|
let overlapText = "";
|
|
6085
5515
|
let j = i - 1;
|
|
@@ -6107,24 +5537,103 @@ class TextChunkerTask extends Task17 {
|
|
|
6107
5537
|
}
|
|
6108
5538
|
}
|
|
6109
5539
|
if (currentChunk.length > 0) {
|
|
6110
|
-
chunks.push(
|
|
6111
|
-
|
|
6112
|
-
index,
|
|
5540
|
+
chunks.push({
|
|
5541
|
+
text: currentChunk.trim(),
|
|
6113
5542
|
startChar: currentStartChar,
|
|
6114
|
-
endChar: currentStartChar + currentChunk.length
|
|
6115
|
-
length: currentChunk.trim().length
|
|
5543
|
+
endChar: currentStartChar + currentChunk.length
|
|
6116
5544
|
});
|
|
6117
5545
|
}
|
|
6118
|
-
return
|
|
5546
|
+
return chunks;
|
|
6119
5547
|
}
|
|
6120
5548
|
}
|
|
6121
5549
|
var textChunker = (input, config) => {
|
|
6122
5550
|
return new TextChunkerTask(config).run(input);
|
|
6123
5551
|
};
|
|
6124
|
-
|
|
5552
|
+
Workflow31.prototype.textChunker = CreateWorkflow31(TextChunkerTask);
|
|
5553
|
+
|
|
5554
|
+
// src/task/TextClassificationTask.ts
|
|
5555
|
+
import { CreateWorkflow as CreateWorkflow32, Workflow as Workflow32 } from "@workglow/task-graph";
|
|
5556
|
+
var modelSchema22 = TypeModel("model:TextClassificationTask");
|
|
5557
|
+
var TextClassificationInputSchema = {
|
|
5558
|
+
type: "object",
|
|
5559
|
+
properties: {
|
|
5560
|
+
text: {
|
|
5561
|
+
type: "string",
|
|
5562
|
+
title: "Text",
|
|
5563
|
+
description: "The text to classify"
|
|
5564
|
+
},
|
|
5565
|
+
candidateLabels: {
|
|
5566
|
+
type: "array",
|
|
5567
|
+
items: {
|
|
5568
|
+
type: "string"
|
|
5569
|
+
},
|
|
5570
|
+
title: "Candidate Labels",
|
|
5571
|
+
description: "List of candidate labels (optional, if provided uses zero-shot classification)",
|
|
5572
|
+
"x-ui-group": "Configuration"
|
|
5573
|
+
},
|
|
5574
|
+
maxCategories: {
|
|
5575
|
+
type: "number",
|
|
5576
|
+
minimum: 1,
|
|
5577
|
+
maximum: 1000,
|
|
5578
|
+
default: 5,
|
|
5579
|
+
title: "Max Categories",
|
|
5580
|
+
description: "The maximum number of categories to return",
|
|
5581
|
+
"x-ui-group": "Configuration"
|
|
5582
|
+
},
|
|
5583
|
+
model: modelSchema22
|
|
5584
|
+
},
|
|
5585
|
+
required: ["text", "model"],
|
|
5586
|
+
additionalProperties: false
|
|
5587
|
+
};
|
|
5588
|
+
var TextClassificationOutputSchema = {
|
|
5589
|
+
type: "object",
|
|
5590
|
+
properties: {
|
|
5591
|
+
categories: {
|
|
5592
|
+
type: "array",
|
|
5593
|
+
items: {
|
|
5594
|
+
type: "object",
|
|
5595
|
+
properties: {
|
|
5596
|
+
label: {
|
|
5597
|
+
type: "string",
|
|
5598
|
+
title: "Label",
|
|
5599
|
+
description: "The name of the category"
|
|
5600
|
+
},
|
|
5601
|
+
score: {
|
|
5602
|
+
type: "number",
|
|
5603
|
+
title: "Score",
|
|
5604
|
+
description: "The confidence score for this category"
|
|
5605
|
+
}
|
|
5606
|
+
},
|
|
5607
|
+
required: ["label", "score"],
|
|
5608
|
+
additionalProperties: false
|
|
5609
|
+
},
|
|
5610
|
+
title: "Categories",
|
|
5611
|
+
description: "The classification categories with their scores"
|
|
5612
|
+
}
|
|
5613
|
+
},
|
|
5614
|
+
required: ["categories"],
|
|
5615
|
+
additionalProperties: false
|
|
5616
|
+
};
|
|
5617
|
+
|
|
5618
|
+
class TextClassificationTask extends AiTask {
|
|
5619
|
+
static type = "TextClassificationTask";
|
|
5620
|
+
static category = "AI Text Model";
|
|
5621
|
+
static title = "Text Classifier";
|
|
5622
|
+
static description = "Classifies text into categories using language models. Supports zero-shot classification when candidate labels are provided.";
|
|
5623
|
+
static inputSchema() {
|
|
5624
|
+
return TextClassificationInputSchema;
|
|
5625
|
+
}
|
|
5626
|
+
static outputSchema() {
|
|
5627
|
+
return TextClassificationOutputSchema;
|
|
5628
|
+
}
|
|
5629
|
+
}
|
|
5630
|
+
var textClassification = (input, config) => {
|
|
5631
|
+
return new TextClassificationTask(config).run(input);
|
|
5632
|
+
};
|
|
5633
|
+
Workflow32.prototype.textClassification = CreateWorkflow32(TextClassificationTask);
|
|
6125
5634
|
|
|
6126
5635
|
// src/task/TextFillMaskTask.ts
|
|
6127
|
-
import { CreateWorkflow as
|
|
5636
|
+
import { CreateWorkflow as CreateWorkflow33, Workflow as Workflow33 } from "@workglow/task-graph";
|
|
6128
5637
|
var modelSchema23 = TypeModel("model:TextFillMaskTask");
|
|
6129
5638
|
var TextFillMaskInputSchema = {
|
|
6130
5639
|
type: "object",
|
|
@@ -6189,10 +5698,10 @@ class TextFillMaskTask extends AiTask {
|
|
|
6189
5698
|
var textFillMask = (input, config) => {
|
|
6190
5699
|
return new TextFillMaskTask(config).run(input);
|
|
6191
5700
|
};
|
|
6192
|
-
|
|
5701
|
+
Workflow33.prototype.textFillMask = CreateWorkflow33(TextFillMaskTask);
|
|
6193
5702
|
|
|
6194
5703
|
// src/task/TextGenerationTask.ts
|
|
6195
|
-
import { CreateWorkflow as
|
|
5704
|
+
import { CreateWorkflow as CreateWorkflow34, Workflow as Workflow34 } from "@workglow/task-graph";
|
|
6196
5705
|
var generatedTextSchema2 = {
|
|
6197
5706
|
type: "string",
|
|
6198
5707
|
title: "Text",
|
|
@@ -6277,10 +5786,10 @@ class TextGenerationTask extends StreamingAiTask {
|
|
|
6277
5786
|
var textGeneration = (input, config) => {
|
|
6278
5787
|
return new TextGenerationTask(config).run(input);
|
|
6279
5788
|
};
|
|
6280
|
-
|
|
5789
|
+
Workflow34.prototype.textGeneration = CreateWorkflow34(TextGenerationTask);
|
|
6281
5790
|
|
|
6282
5791
|
// src/task/TextLanguageDetectionTask.ts
|
|
6283
|
-
import { CreateWorkflow as
|
|
5792
|
+
import { CreateWorkflow as CreateWorkflow35, Workflow as Workflow35 } from "@workglow/task-graph";
|
|
6284
5793
|
var modelSchema25 = TypeModel("model:TextLanguageDetectionTask");
|
|
6285
5794
|
var TextLanguageDetectionInputSchema = {
|
|
6286
5795
|
type: "object",
|
|
@@ -6348,10 +5857,10 @@ class TextLanguageDetectionTask extends AiTask {
|
|
|
6348
5857
|
var textLanguageDetection = (input, config) => {
|
|
6349
5858
|
return new TextLanguageDetectionTask(config).run(input);
|
|
6350
5859
|
};
|
|
6351
|
-
|
|
5860
|
+
Workflow35.prototype.textLanguageDetection = CreateWorkflow35(TextLanguageDetectionTask);
|
|
6352
5861
|
|
|
6353
5862
|
// src/task/TextQuestionAnswerTask.ts
|
|
6354
|
-
import { CreateWorkflow as
|
|
5863
|
+
import { CreateWorkflow as CreateWorkflow36, Workflow as Workflow36 } from "@workglow/task-graph";
|
|
6355
5864
|
var contextSchema = {
|
|
6356
5865
|
type: "string",
|
|
6357
5866
|
title: "Context",
|
|
@@ -6403,10 +5912,10 @@ class TextQuestionAnswerTask extends StreamingAiTask {
|
|
|
6403
5912
|
var textQuestionAnswer = (input, config) => {
|
|
6404
5913
|
return new TextQuestionAnswerTask(config).run(input);
|
|
6405
5914
|
};
|
|
6406
|
-
|
|
5915
|
+
Workflow36.prototype.textQuestionAnswer = CreateWorkflow36(TextQuestionAnswerTask);
|
|
6407
5916
|
|
|
6408
5917
|
// src/task/TextRewriterTask.ts
|
|
6409
|
-
import { CreateWorkflow as
|
|
5918
|
+
import { CreateWorkflow as CreateWorkflow37, Workflow as Workflow37 } from "@workglow/task-graph";
|
|
6410
5919
|
var modelSchema27 = TypeModel("model:TextRewriterTask");
|
|
6411
5920
|
var TextRewriterInputSchema = {
|
|
6412
5921
|
type: "object",
|
|
@@ -6455,10 +5964,10 @@ class TextRewriterTask extends StreamingAiTask {
|
|
|
6455
5964
|
var textRewriter = (input, config) => {
|
|
6456
5965
|
return new TextRewriterTask(config).run(input);
|
|
6457
5966
|
};
|
|
6458
|
-
|
|
5967
|
+
Workflow37.prototype.textRewriter = CreateWorkflow37(TextRewriterTask);
|
|
6459
5968
|
|
|
6460
5969
|
// src/task/TextTranslationTask.ts
|
|
6461
|
-
import { CreateWorkflow as
|
|
5970
|
+
import { CreateWorkflow as CreateWorkflow38, Workflow as Workflow38 } from "@workglow/task-graph";
|
|
6462
5971
|
var modelSchema28 = TypeModel("model:TextTranslationTask");
|
|
6463
5972
|
var translationTextSchema = {
|
|
6464
5973
|
type: "string",
|
|
@@ -6521,10 +6030,10 @@ class TextTranslationTask extends StreamingAiTask {
|
|
|
6521
6030
|
var textTranslation = (input, config) => {
|
|
6522
6031
|
return new TextTranslationTask(config).run(input);
|
|
6523
6032
|
};
|
|
6524
|
-
|
|
6033
|
+
Workflow38.prototype.textTranslation = CreateWorkflow38(TextTranslationTask);
|
|
6525
6034
|
|
|
6526
6035
|
// src/task/ToolCallingTask.ts
|
|
6527
|
-
import { CreateWorkflow as
|
|
6036
|
+
import { CreateWorkflow as CreateWorkflow39, getTaskConstructors, Workflow as Workflow39 } from "@workglow/task-graph";
|
|
6528
6037
|
import { makeFingerprint } from "@workglow/util";
|
|
6529
6038
|
function taskTypesToTools(taskNames, registry) {
|
|
6530
6039
|
const constructors = getTaskConstructors(registry);
|
|
@@ -6756,16 +6265,16 @@ class ToolCallingTask extends StreamingAiTask {
|
|
|
6756
6265
|
var toolCalling = (input, config) => {
|
|
6757
6266
|
return new ToolCallingTask(config).run(input);
|
|
6758
6267
|
};
|
|
6759
|
-
|
|
6268
|
+
Workflow39.prototype.toolCalling = CreateWorkflow39(ToolCallingTask);
|
|
6760
6269
|
|
|
6761
6270
|
// src/task/TopicSegmenterTask.ts
|
|
6762
|
-
import { CreateWorkflow as
|
|
6271
|
+
import { CreateWorkflow as CreateWorkflow40, Task as Task15, Workflow as Workflow40 } from "@workglow/task-graph";
|
|
6763
6272
|
var SegmentationMethod = {
|
|
6764
6273
|
HEURISTIC: "heuristic",
|
|
6765
6274
|
EMBEDDING_SIMILARITY: "embedding-similarity",
|
|
6766
6275
|
HYBRID: "hybrid"
|
|
6767
6276
|
};
|
|
6768
|
-
var
|
|
6277
|
+
var inputSchema13 = {
|
|
6769
6278
|
type: "object",
|
|
6770
6279
|
properties: {
|
|
6771
6280
|
text: {
|
|
@@ -6806,7 +6315,7 @@ var inputSchema16 = {
|
|
|
6806
6315
|
required: ["text"],
|
|
6807
6316
|
additionalProperties: false
|
|
6808
6317
|
};
|
|
6809
|
-
var
|
|
6318
|
+
var outputSchema13 = {
|
|
6810
6319
|
type: "object",
|
|
6811
6320
|
properties: {
|
|
6812
6321
|
segments: {
|
|
@@ -6834,7 +6343,7 @@ var outputSchema16 = {
|
|
|
6834
6343
|
additionalProperties: false
|
|
6835
6344
|
};
|
|
6836
6345
|
|
|
6837
|
-
class TopicSegmenterTask extends
|
|
6346
|
+
class TopicSegmenterTask extends Task15 {
|
|
6838
6347
|
static type = "TopicSegmenterTask";
|
|
6839
6348
|
static category = "Document";
|
|
6840
6349
|
static title = "Topic Segmenter";
|
|
@@ -6842,10 +6351,10 @@ class TopicSegmenterTask extends Task18 {
|
|
|
6842
6351
|
static cacheable = true;
|
|
6843
6352
|
static EMBEDDING_DIMENSIONS = 256;
|
|
6844
6353
|
static inputSchema() {
|
|
6845
|
-
return
|
|
6354
|
+
return inputSchema13;
|
|
6846
6355
|
}
|
|
6847
6356
|
static outputSchema() {
|
|
6848
|
-
return
|
|
6357
|
+
return outputSchema13;
|
|
6849
6358
|
}
|
|
6850
6359
|
async execute(input, context) {
|
|
6851
6360
|
const {
|
|
@@ -7039,10 +6548,10 @@ class TopicSegmenterTask extends Task18 {
|
|
|
7039
6548
|
var topicSegmenter = (input, config) => {
|
|
7040
6549
|
return new TopicSegmenterTask(config).run(input);
|
|
7041
6550
|
};
|
|
7042
|
-
|
|
6551
|
+
Workflow40.prototype.topicSegmenter = CreateWorkflow40(TopicSegmenterTask);
|
|
7043
6552
|
|
|
7044
6553
|
// src/task/UnloadModelTask.ts
|
|
7045
|
-
import { CreateWorkflow as
|
|
6554
|
+
import { CreateWorkflow as CreateWorkflow41, Workflow as Workflow41 } from "@workglow/task-graph";
|
|
7046
6555
|
var modelSchema30 = TypeModel("model");
|
|
7047
6556
|
var UnloadModelInputSchema = {
|
|
7048
6557
|
type: "object",
|
|
@@ -7077,27 +6586,27 @@ class UnloadModelTask extends AiTask {
|
|
|
7077
6586
|
var unloadModel = (input, config) => {
|
|
7078
6587
|
return new UnloadModelTask(config).run(input);
|
|
7079
6588
|
};
|
|
7080
|
-
|
|
6589
|
+
Workflow41.prototype.unloadModel = CreateWorkflow41(UnloadModelTask);
|
|
7081
6590
|
|
|
7082
6591
|
// src/task/VectorQuantizeTask.ts
|
|
7083
|
-
import { CreateWorkflow as
|
|
6592
|
+
import { CreateWorkflow as CreateWorkflow42, Task as Task16, Workflow as Workflow42 } from "@workglow/task-graph";
|
|
7084
6593
|
import {
|
|
7085
6594
|
normalizeNumberArray,
|
|
7086
6595
|
TensorType,
|
|
7087
|
-
TypedArraySchema as
|
|
6596
|
+
TypedArraySchema as TypedArraySchema5
|
|
7088
6597
|
} from "@workglow/util/schema";
|
|
7089
|
-
var
|
|
6598
|
+
var inputSchema14 = {
|
|
7090
6599
|
type: "object",
|
|
7091
6600
|
properties: {
|
|
7092
6601
|
vector: {
|
|
7093
6602
|
anyOf: [
|
|
7094
|
-
|
|
6603
|
+
TypedArraySchema5({
|
|
7095
6604
|
title: "Vector",
|
|
7096
6605
|
description: "The vector to quantize"
|
|
7097
6606
|
}),
|
|
7098
6607
|
{
|
|
7099
6608
|
type: "array",
|
|
7100
|
-
items:
|
|
6609
|
+
items: TypedArraySchema5({
|
|
7101
6610
|
title: "Vector",
|
|
7102
6611
|
description: "Vector to quantize"
|
|
7103
6612
|
})
|
|
@@ -7123,18 +6632,18 @@ var inputSchema17 = {
|
|
|
7123
6632
|
required: ["vector", "targetType"],
|
|
7124
6633
|
additionalProperties: false
|
|
7125
6634
|
};
|
|
7126
|
-
var
|
|
6635
|
+
var outputSchema14 = {
|
|
7127
6636
|
type: "object",
|
|
7128
6637
|
properties: {
|
|
7129
6638
|
vector: {
|
|
7130
6639
|
anyOf: [
|
|
7131
|
-
|
|
6640
|
+
TypedArraySchema5({
|
|
7132
6641
|
title: "Quantized Vector",
|
|
7133
6642
|
description: "The quantized vector"
|
|
7134
6643
|
}),
|
|
7135
6644
|
{
|
|
7136
6645
|
type: "array",
|
|
7137
|
-
items:
|
|
6646
|
+
items: TypedArraySchema5({
|
|
7138
6647
|
title: "Quantized Vector",
|
|
7139
6648
|
description: "Quantized vector"
|
|
7140
6649
|
})
|
|
@@ -7160,19 +6669,22 @@ var outputSchema17 = {
|
|
|
7160
6669
|
additionalProperties: false
|
|
7161
6670
|
};
|
|
7162
6671
|
|
|
7163
|
-
class VectorQuantizeTask extends
|
|
6672
|
+
class VectorQuantizeTask extends Task16 {
|
|
7164
6673
|
static type = "VectorQuantizeTask";
|
|
7165
6674
|
static category = "Vector";
|
|
7166
6675
|
static title = "Quantize";
|
|
7167
6676
|
static description = "Quantize vectors to reduce storage and improve performance";
|
|
7168
6677
|
static cacheable = true;
|
|
7169
6678
|
static inputSchema() {
|
|
7170
|
-
return
|
|
6679
|
+
return inputSchema14;
|
|
7171
6680
|
}
|
|
7172
6681
|
static outputSchema() {
|
|
7173
|
-
return
|
|
6682
|
+
return outputSchema14;
|
|
6683
|
+
}
|
|
6684
|
+
async execute(input) {
|
|
6685
|
+
return this.executePreview(input);
|
|
7174
6686
|
}
|
|
7175
|
-
async
|
|
6687
|
+
async executePreview(input) {
|
|
7176
6688
|
const { vector, targetType, normalize = true } = input;
|
|
7177
6689
|
const isArray = Array.isArray(vector);
|
|
7178
6690
|
const vectors = isArray ? vector : [vector];
|
|
@@ -7260,15 +6772,15 @@ class VectorQuantizeTask extends Task19 {
|
|
|
7260
6772
|
var vectorQuantize = (input, config) => {
|
|
7261
6773
|
return new VectorQuantizeTask(config).run(input);
|
|
7262
6774
|
};
|
|
7263
|
-
|
|
6775
|
+
Workflow42.prototype.vectorQuantize = CreateWorkflow42(VectorQuantizeTask);
|
|
7264
6776
|
|
|
7265
6777
|
// src/task/VectorSimilarityTask.ts
|
|
7266
|
-
import { CreateWorkflow as
|
|
6778
|
+
import { CreateWorkflow as CreateWorkflow43, GraphAsTask, Workflow as Workflow43 } from "@workglow/task-graph";
|
|
7267
6779
|
import {
|
|
7268
6780
|
cosineSimilarity,
|
|
7269
6781
|
hammingSimilarity,
|
|
7270
6782
|
jaccardSimilarity,
|
|
7271
|
-
TypedArraySchema as
|
|
6783
|
+
TypedArraySchema as TypedArraySchema6
|
|
7272
6784
|
} from "@workglow/util/schema";
|
|
7273
6785
|
var SimilarityFn = {
|
|
7274
6786
|
COSINE: "cosine",
|
|
@@ -7283,13 +6795,13 @@ var similarityFunctions = {
|
|
|
7283
6795
|
var SimilarityInputSchema = {
|
|
7284
6796
|
type: "object",
|
|
7285
6797
|
properties: {
|
|
7286
|
-
query:
|
|
6798
|
+
query: TypedArraySchema6({
|
|
7287
6799
|
title: "Query",
|
|
7288
6800
|
description: "Query vector to compare against"
|
|
7289
6801
|
}),
|
|
7290
6802
|
vectors: {
|
|
7291
6803
|
type: "array",
|
|
7292
|
-
items:
|
|
6804
|
+
items: TypedArraySchema6({
|
|
7293
6805
|
title: "Input",
|
|
7294
6806
|
description: "Array of vectors to compare against the query"
|
|
7295
6807
|
})
|
|
@@ -7317,7 +6829,7 @@ var SimilarityOutputSchema = {
|
|
|
7317
6829
|
properties: {
|
|
7318
6830
|
output: {
|
|
7319
6831
|
type: "array",
|
|
7320
|
-
items:
|
|
6832
|
+
items: TypedArraySchema6({
|
|
7321
6833
|
title: "Output",
|
|
7322
6834
|
description: "Ranked output vectors"
|
|
7323
6835
|
})
|
|
@@ -7347,7 +6859,15 @@ class VectorSimilarityTask extends GraphAsTask {
|
|
|
7347
6859
|
static outputSchema() {
|
|
7348
6860
|
return SimilarityOutputSchema;
|
|
7349
6861
|
}
|
|
7350
|
-
async
|
|
6862
|
+
async execute(input) {
|
|
6863
|
+
return this.executePreview(input);
|
|
6864
|
+
}
|
|
6865
|
+
async executePreview({
|
|
6866
|
+
query,
|
|
6867
|
+
vectors,
|
|
6868
|
+
method,
|
|
6869
|
+
topK
|
|
6870
|
+
}) {
|
|
7351
6871
|
let similarities = [];
|
|
7352
6872
|
const fnName = method;
|
|
7353
6873
|
const fn = similarityFunctions[fnName];
|
|
@@ -7369,7 +6889,7 @@ class VectorSimilarityTask extends GraphAsTask {
|
|
|
7369
6889
|
var similarity = (input, config) => {
|
|
7370
6890
|
return new VectorSimilarityTask(config).run(input);
|
|
7371
6891
|
};
|
|
7372
|
-
|
|
6892
|
+
Workflow43.prototype.similarity = CreateWorkflow43(VectorSimilarityTask);
|
|
7373
6893
|
// src/task/MessageConversion.ts
|
|
7374
6894
|
function getInputMessages(input) {
|
|
7375
6895
|
const messages = input.messages;
|
|
@@ -7523,14 +7043,11 @@ var registerAiTasks = () => {
|
|
|
7523
7043
|
const tasks = [
|
|
7524
7044
|
AiChatTask,
|
|
7525
7045
|
BackgroundRemovalTask,
|
|
7526
|
-
ChunkToVectorTask,
|
|
7527
7046
|
CountTokensTask,
|
|
7528
7047
|
ContextBuilderTask,
|
|
7529
7048
|
DocumentEnricherTask,
|
|
7530
7049
|
DocumentUpsertTask,
|
|
7531
7050
|
ChunkRetrievalTask,
|
|
7532
|
-
ChunkVectorHybridSearchTask,
|
|
7533
|
-
ChunkVectorSearchTask,
|
|
7534
7051
|
ChunkVectorUpsertTask,
|
|
7535
7052
|
DownloadModelTask,
|
|
7536
7053
|
FaceDetectorTask,
|
|
@@ -7573,7 +7090,6 @@ var registerAiTasks = () => {
|
|
|
7573
7090
|
return tasks;
|
|
7574
7091
|
};
|
|
7575
7092
|
export {
|
|
7576
|
-
vectorStoreSearch,
|
|
7577
7093
|
vectorQuantize,
|
|
7578
7094
|
unloadModel,
|
|
7579
7095
|
topicSegmenter,
|
|
@@ -7615,7 +7131,6 @@ export {
|
|
|
7615
7131
|
imageSegmentation,
|
|
7616
7132
|
imageEmbedding,
|
|
7617
7133
|
imageClassification,
|
|
7618
|
-
hybridSearch,
|
|
7619
7134
|
hierarchyJoin,
|
|
7620
7135
|
hierarchicalChunker,
|
|
7621
7136
|
handLandmarker,
|
|
@@ -7631,7 +7146,6 @@ export {
|
|
|
7631
7146
|
countTokens,
|
|
7632
7147
|
contextBuilder,
|
|
7633
7148
|
chunkVectorUpsert,
|
|
7634
|
-
chunkToVector,
|
|
7635
7149
|
chunkRetrieval,
|
|
7636
7150
|
buildToolDescription,
|
|
7637
7151
|
backgroundRemoval,
|
|
@@ -7750,9 +7264,6 @@ export {
|
|
|
7750
7264
|
ContentBlockSchema,
|
|
7751
7265
|
ChunkingStrategy,
|
|
7752
7266
|
ChunkVectorUpsertTask,
|
|
7753
|
-
ChunkVectorSearchTask,
|
|
7754
|
-
ChunkVectorHybridSearchTask,
|
|
7755
|
-
ChunkToVectorTask,
|
|
7756
7267
|
ChunkRetrievalTask,
|
|
7757
7268
|
ChatMessageSchema,
|
|
7758
7269
|
BackgroundRemovalTask,
|
|
@@ -7767,4 +7278,4 @@ export {
|
|
|
7767
7278
|
AiChatInputSchema
|
|
7768
7279
|
};
|
|
7769
7280
|
|
|
7770
|
-
//# debugId=
|
|
7281
|
+
//# debugId=4A65F3B2D404BCCD64756E2164756E21
|