@workglow/ai 0.2.15 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -15
- package/dist/browser.js +544 -1033
- package/dist/browser.js.map +17 -20
- package/dist/bun.js +544 -1033
- package/dist/bun.js.map +17 -20
- package/dist/node.js +544 -1033
- package/dist/node.js.map +17 -20
- package/dist/provider/AiProvider.d.ts +8 -8
- package/dist/provider/AiProvider.d.ts.map +1 -1
- package/dist/provider/AiProviderRegistry.d.ts +16 -16
- package/dist/provider/AiProviderRegistry.d.ts.map +1 -1
- package/dist/task/ChunkRetrievalTask.d.ts +32 -49
- package/dist/task/ChunkRetrievalTask.d.ts.map +1 -1
- package/dist/task/ChunkVectorUpsertTask.d.ts +107 -24
- package/dist/task/ChunkVectorUpsertTask.d.ts.map +1 -1
- package/dist/task/ContextBuilderTask.d.ts +3 -2
- package/dist/task/ContextBuilderTask.d.ts.map +1 -1
- package/dist/task/HierarchyJoinTask.d.ts +44 -42
- package/dist/task/HierarchyJoinTask.d.ts.map +1 -1
- package/dist/task/QueryExpanderTask.d.ts +5 -31
- package/dist/task/QueryExpanderTask.d.ts.map +1 -1
- package/dist/task/RerankerTask.d.ts +7 -89
- package/dist/task/RerankerTask.d.ts.map +1 -1
- package/dist/task/TextChunkerTask.d.ts +139 -37
- package/dist/task/TextChunkerTask.d.ts.map +1 -1
- package/dist/task/VectorQuantizeTask.d.ts +2 -1
- package/dist/task/VectorQuantizeTask.d.ts.map +1 -1
- package/dist/task/VectorSimilarityTask.d.ts +2 -4
- package/dist/task/VectorSimilarityTask.d.ts.map +1 -1
- package/dist/task/base/AiTask.d.ts +4 -4
- package/dist/task/base/AiTask.d.ts.map +1 -1
- package/dist/task/index.d.ts +1 -7
- package/dist/task/index.d.ts.map +1 -1
- package/dist/worker.js +25 -26
- package/dist/worker.js.map +4 -4
- package/package.json +11 -11
- package/dist/task/ChunkToVectorTask.d.ts +0 -210
- package/dist/task/ChunkToVectorTask.d.ts.map +0 -1
- package/dist/task/ChunkVectorHybridSearchTask.d.ts +0 -167
- package/dist/task/ChunkVectorHybridSearchTask.d.ts.map +0 -1
- package/dist/task/ChunkVectorSearchTask.d.ts +0 -139
- package/dist/task/ChunkVectorSearchTask.d.ts.map +0 -1
package/dist/browser.js
CHANGED
|
@@ -14,7 +14,7 @@ import { globalServiceRegistry, WORKER_MANAGER } from "@workglow/util/worker";
|
|
|
14
14
|
class AiProviderRegistry {
|
|
15
15
|
runFnRegistry = new Map;
|
|
16
16
|
streamFnRegistry = new Map;
|
|
17
|
-
|
|
17
|
+
previewRunFnRegistry = new Map;
|
|
18
18
|
providers = new Map;
|
|
19
19
|
strategyResolvers = new Map;
|
|
20
20
|
defaultStrategy;
|
|
@@ -30,7 +30,7 @@ class AiProviderRegistry {
|
|
|
30
30
|
for (const [, providerMap] of this.streamFnRegistry) {
|
|
31
31
|
providerMap.delete(name);
|
|
32
32
|
}
|
|
33
|
-
for (const [, providerMap] of this.
|
|
33
|
+
for (const [, providerMap] of this.previewRunFnRegistry) {
|
|
34
34
|
providerMap.delete(name);
|
|
35
35
|
}
|
|
36
36
|
}
|
|
@@ -108,25 +108,24 @@ class AiProviderRegistry {
|
|
|
108
108
|
const taskTypeMap = this.streamFnRegistry.get(taskType);
|
|
109
109
|
return taskTypeMap?.get(modelProvider);
|
|
110
110
|
}
|
|
111
|
-
|
|
112
|
-
const
|
|
111
|
+
registerAsWorkerPreviewRunFn(modelProvider, taskType) {
|
|
112
|
+
const previewFn = async (input, model) => {
|
|
113
113
|
const workerManager = globalServiceRegistry.get(WORKER_MANAGER);
|
|
114
|
-
return workerManager.
|
|
114
|
+
return workerManager.callWorkerPreviewFunction(modelProvider, taskType, [
|
|
115
115
|
input,
|
|
116
|
-
output,
|
|
117
116
|
model
|
|
118
117
|
]);
|
|
119
118
|
};
|
|
120
|
-
this.
|
|
119
|
+
this.registerPreviewRunFn(modelProvider, taskType, previewFn);
|
|
121
120
|
}
|
|
122
|
-
|
|
123
|
-
if (!this.
|
|
124
|
-
this.
|
|
121
|
+
registerPreviewRunFn(modelProvider, taskType, previewRunFn) {
|
|
122
|
+
if (!this.previewRunFnRegistry.has(taskType)) {
|
|
123
|
+
this.previewRunFnRegistry.set(taskType, new Map);
|
|
125
124
|
}
|
|
126
|
-
this.
|
|
125
|
+
this.previewRunFnRegistry.get(taskType).set(modelProvider, previewRunFn);
|
|
127
126
|
}
|
|
128
|
-
|
|
129
|
-
const taskTypeMap = this.
|
|
127
|
+
getPreviewRunFn(modelProvider, taskType) {
|
|
128
|
+
const taskTypeMap = this.previewRunFnRegistry.get(taskType);
|
|
130
129
|
return taskTypeMap?.get(modelProvider);
|
|
131
130
|
}
|
|
132
131
|
getDirectRunFn(modelProvider, taskType) {
|
|
@@ -721,11 +720,11 @@ function resolveAiProviderGpuQueueConcurrency(concurrency) {
|
|
|
721
720
|
class AiProvider {
|
|
722
721
|
tasks;
|
|
723
722
|
streamTasks;
|
|
724
|
-
|
|
725
|
-
constructor(tasks, streamTasks,
|
|
723
|
+
previewTasks;
|
|
724
|
+
constructor(tasks, streamTasks, previewTasks) {
|
|
726
725
|
this.tasks = tasks;
|
|
727
726
|
this.streamTasks = streamTasks;
|
|
728
|
-
this.
|
|
727
|
+
this.previewTasks = previewTasks;
|
|
729
728
|
}
|
|
730
729
|
get supportedTaskTypes() {
|
|
731
730
|
return this.taskTypes;
|
|
@@ -736,8 +735,8 @@ class AiProvider {
|
|
|
736
735
|
getStreamFn(taskType) {
|
|
737
736
|
return this.streamTasks?.[taskType];
|
|
738
737
|
}
|
|
739
|
-
|
|
740
|
-
return this.
|
|
738
|
+
getPreviewRunFn(taskType) {
|
|
739
|
+
return this.previewTasks?.[taskType];
|
|
741
740
|
}
|
|
742
741
|
async register(options = {}) {
|
|
743
742
|
const isInline = !!this.tasks;
|
|
@@ -765,7 +764,7 @@ class AiProvider {
|
|
|
765
764
|
for (const taskType of this.taskTypes) {
|
|
766
765
|
registry.registerAsWorkerRunFn(this.name, taskType);
|
|
767
766
|
registry.registerAsWorkerStreamFn(this.name, taskType);
|
|
768
|
-
registry.
|
|
767
|
+
registry.registerAsWorkerPreviewRunFn(this.name, taskType);
|
|
769
768
|
}
|
|
770
769
|
} else {
|
|
771
770
|
for (const [taskType, fn] of Object.entries(this.tasks)) {
|
|
@@ -777,9 +776,9 @@ class AiProvider {
|
|
|
777
776
|
}
|
|
778
777
|
}
|
|
779
778
|
}
|
|
780
|
-
if (this.
|
|
781
|
-
for (const [taskType, fn] of Object.entries(this.
|
|
782
|
-
registry.
|
|
779
|
+
if (this.previewTasks) {
|
|
780
|
+
for (const [taskType, fn] of Object.entries(this.previewTasks)) {
|
|
781
|
+
registry.registerPreviewRunFn(this.name, taskType, fn);
|
|
783
782
|
}
|
|
784
783
|
}
|
|
785
784
|
registry.registerProvider(this);
|
|
@@ -802,9 +801,9 @@ class AiProvider {
|
|
|
802
801
|
workerServer.registerStreamFunction(taskType, fn);
|
|
803
802
|
}
|
|
804
803
|
}
|
|
805
|
-
if (this.
|
|
806
|
-
for (const [taskType, fn] of Object.entries(this.
|
|
807
|
-
workerServer.
|
|
804
|
+
if (this.previewTasks) {
|
|
805
|
+
for (const [taskType, fn] of Object.entries(this.previewTasks)) {
|
|
806
|
+
workerServer.registerPreviewFunction(taskType, fn);
|
|
808
807
|
}
|
|
809
808
|
}
|
|
810
809
|
}
|
|
@@ -1111,16 +1110,16 @@ class AiTask extends Task {
|
|
|
1111
1110
|
const model = input.model;
|
|
1112
1111
|
return model?.provider;
|
|
1113
1112
|
}
|
|
1114
|
-
async
|
|
1113
|
+
async executePreview(input, context) {
|
|
1115
1114
|
const model = input.model;
|
|
1116
1115
|
if (model && typeof model === "object" && model.provider) {
|
|
1117
1116
|
const taskType = this.constructor.runtype ?? this.constructor.type;
|
|
1118
|
-
const
|
|
1119
|
-
if (
|
|
1120
|
-
return
|
|
1117
|
+
const previewFn = getAiProviderRegistry().getPreviewRunFn(model.provider, taskType);
|
|
1118
|
+
if (previewFn) {
|
|
1119
|
+
return previewFn(input, model);
|
|
1121
1120
|
}
|
|
1122
1121
|
}
|
|
1123
|
-
return super.
|
|
1122
|
+
return super.executePreview(input, context);
|
|
1124
1123
|
}
|
|
1125
1124
|
async validateInput(input) {
|
|
1126
1125
|
const inputSchema = this.inputSchema();
|
|
@@ -1707,7 +1706,7 @@ var inputSchema = {
|
|
|
1707
1706
|
title: "Knowledge Base",
|
|
1708
1707
|
description: "The knowledge base instance to search in"
|
|
1709
1708
|
}),
|
|
1710
|
-
query:
|
|
1709
|
+
query: {
|
|
1711
1710
|
oneOf: [
|
|
1712
1711
|
{ type: "string" },
|
|
1713
1712
|
TypedArraySchema2({
|
|
@@ -1716,12 +1715,19 @@ var inputSchema = {
|
|
|
1716
1715
|
})
|
|
1717
1716
|
],
|
|
1718
1717
|
title: "Query",
|
|
1719
|
-
description: "Query string or pre-computed query vector"
|
|
1720
|
-
}
|
|
1718
|
+
description: "Query string (requires `model`) or pre-computed query vector"
|
|
1719
|
+
},
|
|
1721
1720
|
model: TypeModel("model:TextEmbeddingTask", {
|
|
1722
1721
|
title: "Model",
|
|
1723
1722
|
description: "Text embedding model to use for query embedding (required when query is a string)"
|
|
1724
1723
|
}),
|
|
1724
|
+
method: {
|
|
1725
|
+
type: "string",
|
|
1726
|
+
enum: ["similarity", "hybrid"],
|
|
1727
|
+
title: "Retrieval Method",
|
|
1728
|
+
description: "Retrieval strategy: 'similarity' (vector only) or 'hybrid' (vector + full-text).",
|
|
1729
|
+
default: "similarity"
|
|
1730
|
+
},
|
|
1725
1731
|
topK: {
|
|
1726
1732
|
type: "number",
|
|
1727
1733
|
title: "Top K",
|
|
@@ -1742,6 +1748,14 @@ var inputSchema = {
|
|
|
1742
1748
|
maximum: 1,
|
|
1743
1749
|
default: 0
|
|
1744
1750
|
},
|
|
1751
|
+
vectorWeight: {
|
|
1752
|
+
type: "number",
|
|
1753
|
+
title: "Vector Weight",
|
|
1754
|
+
description: "For hybrid method: weight for vector similarity (0-1), remainder goes to text relevance",
|
|
1755
|
+
minimum: 0,
|
|
1756
|
+
maximum: 1,
|
|
1757
|
+
default: 0.7
|
|
1758
|
+
},
|
|
1745
1759
|
returnVectors: {
|
|
1746
1760
|
type: "boolean",
|
|
1747
1761
|
title: "Return Vectors",
|
|
@@ -1806,7 +1820,7 @@ var outputSchema = {
|
|
|
1806
1820
|
title: "Count",
|
|
1807
1821
|
description: "Number of results returned"
|
|
1808
1822
|
},
|
|
1809
|
-
query:
|
|
1823
|
+
query: {
|
|
1810
1824
|
oneOf: [
|
|
1811
1825
|
{ type: "string" },
|
|
1812
1826
|
TypedArraySchema2({
|
|
@@ -1816,7 +1830,7 @@ var outputSchema = {
|
|
|
1816
1830
|
],
|
|
1817
1831
|
title: "Query",
|
|
1818
1832
|
description: "The query used for retrieval (pass-through)"
|
|
1819
|
-
}
|
|
1833
|
+
}
|
|
1820
1834
|
},
|
|
1821
1835
|
required: ["chunks", "chunk_ids", "metadata", "scores", "count", "query"],
|
|
1822
1836
|
additionalProperties: false
|
|
@@ -1826,7 +1840,7 @@ class ChunkRetrievalTask extends Task2 {
|
|
|
1826
1840
|
static type = "ChunkRetrievalTask";
|
|
1827
1841
|
static category = "RAG";
|
|
1828
1842
|
static title = "Chunk Retrieval";
|
|
1829
|
-
static description = "End-to-end retrieval: embed query and search
|
|
1843
|
+
static description = "End-to-end retrieval: embed query (if string) and search the knowledge base. Supports similarity and hybrid methods.";
|
|
1830
1844
|
static cacheable = true;
|
|
1831
1845
|
static inputSchema() {
|
|
1832
1846
|
return inputSchema;
|
|
@@ -1841,332 +1855,46 @@ class ChunkRetrievalTask extends Task2 {
|
|
|
1841
1855
|
topK = 5,
|
|
1842
1856
|
filter,
|
|
1843
1857
|
model,
|
|
1858
|
+
method = "similarity",
|
|
1859
|
+
vectorWeight = 0.7,
|
|
1844
1860
|
scoreThreshold = 0,
|
|
1845
1861
|
returnVectors = false
|
|
1846
1862
|
} = input;
|
|
1847
1863
|
const kb = knowledgeBase;
|
|
1848
|
-
|
|
1849
|
-
if (
|
|
1864
|
+
const queryIsString = typeof query === "string";
|
|
1865
|
+
if (method === "hybrid" && !queryIsString) {
|
|
1866
|
+
throw new Error("Hybrid retrieval requires a string query (it will be embedded and used for full-text search).");
|
|
1867
|
+
}
|
|
1868
|
+
if (method === "hybrid" && !kb.supportsHybridSearch()) {
|
|
1869
|
+
throw new Error("The provided knowledge base does not support hybrid search. Use method: 'similarity' or a backend with hybrid support (e.g., Postgres with pgvector).");
|
|
1870
|
+
}
|
|
1871
|
+
let queryVector;
|
|
1872
|
+
let queryText;
|
|
1873
|
+
if (queryIsString) {
|
|
1850
1874
|
if (!model) {
|
|
1851
1875
|
throw new Error("Model is required when query is a string. Please provide a model with format 'model:TextEmbeddingTask'.");
|
|
1852
1876
|
}
|
|
1877
|
+
queryText = query;
|
|
1853
1878
|
const embeddingTask = context.own(new TextEmbeddingTask);
|
|
1854
1879
|
const embeddingResult = await embeddingTask.run({ text: query, model });
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1880
|
+
const vec = embeddingResult.vector;
|
|
1881
|
+
queryVector = Array.isArray(vec) ? vec[0] : vec;
|
|
1882
|
+
} else if (isTypedArray(query)) {
|
|
1883
|
+
queryVector = query;
|
|
1858
1884
|
} else {
|
|
1859
|
-
throw new Error("Query must be a string
|
|
1860
|
-
}
|
|
1861
|
-
const searchVectors = queryVectors.map((v) => v instanceof Float32Array ? v : new Float32Array(v));
|
|
1862
|
-
const results = [];
|
|
1863
|
-
for (const searchVector of searchVectors) {
|
|
1864
|
-
const res = await kb.similaritySearch(searchVector, {
|
|
1865
|
-
topK,
|
|
1866
|
-
filter,
|
|
1867
|
-
scoreThreshold
|
|
1868
|
-
});
|
|
1869
|
-
results.push(...res);
|
|
1870
|
-
}
|
|
1871
|
-
const chunks = results.map((r) => {
|
|
1872
|
-
const meta = r.metadata;
|
|
1873
|
-
return meta.text || JSON.stringify(meta);
|
|
1874
|
-
});
|
|
1875
|
-
const output = {
|
|
1876
|
-
chunks,
|
|
1877
|
-
chunk_ids: results.map((r) => r.chunk_id),
|
|
1878
|
-
metadata: results.map((r) => r.metadata),
|
|
1879
|
-
scores: results.map((r) => r.score),
|
|
1880
|
-
count: results.length,
|
|
1881
|
-
query
|
|
1882
|
-
};
|
|
1883
|
-
if (returnVectors) {
|
|
1884
|
-
output.vectors = results.map((r) => r.vector);
|
|
1885
|
+
throw new Error("Query must be a string or TypedArray");
|
|
1885
1886
|
}
|
|
1886
|
-
return output;
|
|
1887
|
-
}
|
|
1888
|
-
}
|
|
1889
|
-
var chunkRetrieval = (input, config) => {
|
|
1890
|
-
return new ChunkRetrievalTask(config).run(input);
|
|
1891
|
-
};
|
|
1892
|
-
Workflow3.prototype.chunkRetrieval = CreateWorkflow3(ChunkRetrievalTask);
|
|
1893
|
-
|
|
1894
|
-
// src/task/ChunkToVectorTask.ts
|
|
1895
|
-
import { ChunkRecordSchema } from "@workglow/knowledge-base";
|
|
1896
|
-
import { CreateWorkflow as CreateWorkflow4, Task as Task3, Workflow as Workflow4 } from "@workglow/task-graph";
|
|
1897
|
-
import {
|
|
1898
|
-
TypedArraySchema as TypedArraySchema3
|
|
1899
|
-
} from "@workglow/util/schema";
|
|
1900
|
-
var inputSchema2 = {
|
|
1901
|
-
type: "object",
|
|
1902
|
-
properties: {
|
|
1903
|
-
doc_id: {
|
|
1904
|
-
type: "string",
|
|
1905
|
-
title: "Document ID",
|
|
1906
|
-
description: "The document ID"
|
|
1907
|
-
},
|
|
1908
|
-
doc_title: {
|
|
1909
|
-
type: "string",
|
|
1910
|
-
title: "Document Title",
|
|
1911
|
-
description: "Human-readable title for the source document"
|
|
1912
|
-
},
|
|
1913
|
-
chunks: {
|
|
1914
|
-
type: "array",
|
|
1915
|
-
items: ChunkRecordSchema(),
|
|
1916
|
-
title: "Chunks",
|
|
1917
|
-
description: "Array of chunk records"
|
|
1918
|
-
},
|
|
1919
|
-
vector: {
|
|
1920
|
-
type: "array",
|
|
1921
|
-
items: TypedArraySchema3({
|
|
1922
|
-
title: "Vector",
|
|
1923
|
-
description: "Vector embedding"
|
|
1924
|
-
}),
|
|
1925
|
-
title: "Vectors",
|
|
1926
|
-
description: "Embeddings from TextEmbeddingTask"
|
|
1927
|
-
}
|
|
1928
|
-
},
|
|
1929
|
-
required: ["chunks", "vector"],
|
|
1930
|
-
additionalProperties: false
|
|
1931
|
-
};
|
|
1932
|
-
var outputSchema2 = {
|
|
1933
|
-
type: "object",
|
|
1934
|
-
properties: {
|
|
1935
|
-
ids: {
|
|
1936
|
-
type: "array",
|
|
1937
|
-
items: { type: "string" },
|
|
1938
|
-
title: "IDs",
|
|
1939
|
-
description: "Chunk IDs for vector store"
|
|
1940
|
-
},
|
|
1941
|
-
vectors: {
|
|
1942
|
-
type: "array",
|
|
1943
|
-
items: TypedArraySchema3({
|
|
1944
|
-
title: "Vector",
|
|
1945
|
-
description: "Vector embedding"
|
|
1946
|
-
}),
|
|
1947
|
-
title: "Vectors",
|
|
1948
|
-
description: "Vector embeddings"
|
|
1949
|
-
},
|
|
1950
|
-
metadata: {
|
|
1951
|
-
type: "array",
|
|
1952
|
-
items: {
|
|
1953
|
-
type: "object",
|
|
1954
|
-
title: "Metadata",
|
|
1955
|
-
description: "Metadata for vector store"
|
|
1956
|
-
},
|
|
1957
|
-
title: "Metadata",
|
|
1958
|
-
description: "Metadata for each vector"
|
|
1959
|
-
},
|
|
1960
|
-
texts: {
|
|
1961
|
-
type: "array",
|
|
1962
|
-
items: { type: "string" },
|
|
1963
|
-
title: "Texts",
|
|
1964
|
-
description: "Chunk texts (for reference)"
|
|
1965
|
-
}
|
|
1966
|
-
},
|
|
1967
|
-
required: ["ids", "vectors", "metadata", "texts"],
|
|
1968
|
-
additionalProperties: false
|
|
1969
|
-
};
|
|
1970
|
-
|
|
1971
|
-
class ChunkToVectorTask extends Task3 {
|
|
1972
|
-
static type = "ChunkToVectorTask";
|
|
1973
|
-
static category = "Document";
|
|
1974
|
-
static title = "Chunk to Vector";
|
|
1975
|
-
static description = "Transform chunks and embeddings to vector store format";
|
|
1976
|
-
static cacheable = true;
|
|
1977
|
-
static inputSchema() {
|
|
1978
|
-
return inputSchema2;
|
|
1979
|
-
}
|
|
1980
|
-
static outputSchema() {
|
|
1981
|
-
return outputSchema2;
|
|
1982
|
-
}
|
|
1983
|
-
async execute(input, context) {
|
|
1984
|
-
const { chunks, vector, doc_title } = input;
|
|
1985
|
-
const chunkArray = chunks;
|
|
1986
|
-
if (!chunkArray || !vector) {
|
|
1987
|
-
throw new Error("Both chunks and vector are required");
|
|
1988
|
-
}
|
|
1989
|
-
if (chunkArray.length !== vector.length) {
|
|
1990
|
-
throw new Error(`Mismatch: ${chunkArray.length} chunks but ${vector.length} vectors`);
|
|
1991
|
-
}
|
|
1992
|
-
const ids = [];
|
|
1993
|
-
const metadata = [];
|
|
1994
|
-
const texts = [];
|
|
1995
|
-
for (let i = 0;i < chunkArray.length; i++) {
|
|
1996
|
-
const chunk = chunkArray[i];
|
|
1997
|
-
ids.push(chunk.chunkId);
|
|
1998
|
-
texts.push(chunk.text);
|
|
1999
|
-
metadata.push({
|
|
2000
|
-
doc_id: chunk.doc_id,
|
|
2001
|
-
chunkId: chunk.chunkId,
|
|
2002
|
-
leafNodeId: chunk.nodePath[chunk.nodePath.length - 1],
|
|
2003
|
-
depth: chunk.depth,
|
|
2004
|
-
text: chunk.text,
|
|
2005
|
-
nodePath: chunk.nodePath,
|
|
2006
|
-
...doc_title ? { doc_title } : {},
|
|
2007
|
-
...chunk.summary ? { summary: chunk.summary } : {},
|
|
2008
|
-
...chunk.entities ? { entities: chunk.entities } : {}
|
|
2009
|
-
});
|
|
2010
|
-
}
|
|
2011
|
-
return {
|
|
2012
|
-
ids,
|
|
2013
|
-
vectors: vector,
|
|
2014
|
-
metadata,
|
|
2015
|
-
texts
|
|
2016
|
-
};
|
|
2017
|
-
}
|
|
2018
|
-
}
|
|
2019
|
-
var chunkToVector = (input, config) => {
|
|
2020
|
-
return new ChunkToVectorTask(config).run(input);
|
|
2021
|
-
};
|
|
2022
|
-
Workflow4.prototype.chunkToVector = CreateWorkflow4(ChunkToVectorTask);
|
|
2023
|
-
|
|
2024
|
-
// src/task/ChunkVectorHybridSearchTask.ts
|
|
2025
|
-
import { TypeKnowledgeBase as TypeKnowledgeBase2 } from "@workglow/knowledge-base";
|
|
2026
|
-
import { CreateWorkflow as CreateWorkflow5, Task as Task4, Workflow as Workflow5 } from "@workglow/task-graph";
|
|
2027
|
-
import {
|
|
2028
|
-
TypedArraySchema as TypedArraySchema4
|
|
2029
|
-
} from "@workglow/util/schema";
|
|
2030
|
-
var inputSchema3 = {
|
|
2031
|
-
type: "object",
|
|
2032
|
-
properties: {
|
|
2033
|
-
knowledgeBase: TypeKnowledgeBase2({
|
|
2034
|
-
title: "Knowledge Base",
|
|
2035
|
-
description: "The knowledge base instance to search in (must support hybridSearch)"
|
|
2036
|
-
}),
|
|
2037
|
-
queryVector: TypedArraySchema4({
|
|
2038
|
-
title: "Query Vector",
|
|
2039
|
-
description: "The query vector for semantic search"
|
|
2040
|
-
}),
|
|
2041
|
-
queryText: {
|
|
2042
|
-
type: "string",
|
|
2043
|
-
title: "Query Text",
|
|
2044
|
-
description: "The query text for full-text search"
|
|
2045
|
-
},
|
|
2046
|
-
topK: {
|
|
2047
|
-
type: "number",
|
|
2048
|
-
title: "Top K",
|
|
2049
|
-
description: "Number of top results to return",
|
|
2050
|
-
minimum: 1,
|
|
2051
|
-
default: 10
|
|
2052
|
-
},
|
|
2053
|
-
filter: {
|
|
2054
|
-
type: "object",
|
|
2055
|
-
title: "Metadata Filter",
|
|
2056
|
-
description: "Filter results by metadata fields"
|
|
2057
|
-
},
|
|
2058
|
-
scoreThreshold: {
|
|
2059
|
-
type: "number",
|
|
2060
|
-
title: "Score Threshold",
|
|
2061
|
-
description: "Minimum combined score threshold (0-1)",
|
|
2062
|
-
minimum: 0,
|
|
2063
|
-
maximum: 1,
|
|
2064
|
-
default: 0
|
|
2065
|
-
},
|
|
2066
|
-
vectorWeight: {
|
|
2067
|
-
type: "number",
|
|
2068
|
-
title: "Vector Weight",
|
|
2069
|
-
description: "Weight for vector similarity (0-1), remainder goes to text relevance",
|
|
2070
|
-
minimum: 0,
|
|
2071
|
-
maximum: 1,
|
|
2072
|
-
default: 0.7
|
|
2073
|
-
},
|
|
2074
|
-
returnVectors: {
|
|
2075
|
-
type: "boolean",
|
|
2076
|
-
title: "Return Vectors",
|
|
2077
|
-
description: "Whether to return vector embeddings in results",
|
|
2078
|
-
default: false
|
|
2079
|
-
}
|
|
2080
|
-
},
|
|
2081
|
-
required: ["knowledgeBase", "queryVector", "queryText"],
|
|
2082
|
-
additionalProperties: false
|
|
2083
|
-
};
|
|
2084
|
-
var outputSchema3 = {
|
|
2085
|
-
type: "object",
|
|
2086
|
-
properties: {
|
|
2087
|
-
chunks: {
|
|
2088
|
-
type: "array",
|
|
2089
|
-
items: { type: "string" },
|
|
2090
|
-
title: "Text Chunks",
|
|
2091
|
-
description: "Retrieved text chunks"
|
|
2092
|
-
},
|
|
2093
|
-
chunk_ids: {
|
|
2094
|
-
type: "array",
|
|
2095
|
-
items: { type: "string" },
|
|
2096
|
-
title: "Chunk IDs",
|
|
2097
|
-
description: "IDs of retrieved chunks"
|
|
2098
|
-
},
|
|
2099
|
-
metadata: {
|
|
2100
|
-
type: "array",
|
|
2101
|
-
items: {
|
|
2102
|
-
type: "object",
|
|
2103
|
-
title: "Metadata",
|
|
2104
|
-
description: "Metadata of retrieved chunk"
|
|
2105
|
-
},
|
|
2106
|
-
title: "Metadata",
|
|
2107
|
-
description: "Metadata of retrieved chunks"
|
|
2108
|
-
},
|
|
2109
|
-
scores: {
|
|
2110
|
-
type: "array",
|
|
2111
|
-
items: { type: "number" },
|
|
2112
|
-
title: "Scores",
|
|
2113
|
-
description: "Combined relevance scores for each result"
|
|
2114
|
-
},
|
|
2115
|
-
vectors: {
|
|
2116
|
-
type: "array",
|
|
2117
|
-
items: TypedArraySchema4({
|
|
2118
|
-
title: "Vector",
|
|
2119
|
-
description: "Vector embedding"
|
|
2120
|
-
}),
|
|
2121
|
-
title: "Vectors",
|
|
2122
|
-
description: "Vector embeddings (if returnVectors is true)"
|
|
2123
|
-
},
|
|
2124
|
-
count: {
|
|
2125
|
-
type: "number",
|
|
2126
|
-
title: "Count",
|
|
2127
|
-
description: "Number of results returned"
|
|
2128
|
-
},
|
|
2129
|
-
query: {
|
|
2130
|
-
type: "string",
|
|
2131
|
-
title: "Query",
|
|
2132
|
-
description: "The text query used for search (pass-through)"
|
|
2133
|
-
}
|
|
2134
|
-
},
|
|
2135
|
-
required: ["chunks", "chunk_ids", "metadata", "scores", "count", "query"],
|
|
2136
|
-
additionalProperties: false
|
|
2137
|
-
};
|
|
2138
|
-
|
|
2139
|
-
class ChunkVectorHybridSearchTask extends Task4 {
|
|
2140
|
-
static type = "ChunkVectorHybridSearchTask";
|
|
2141
|
-
static category = "RAG";
|
|
2142
|
-
static title = "Hybrid Search";
|
|
2143
|
-
static description = "Combined vector + full-text search for improved retrieval";
|
|
2144
|
-
static cacheable = true;
|
|
2145
|
-
static inputSchema() {
|
|
2146
|
-
return inputSchema3;
|
|
2147
|
-
}
|
|
2148
|
-
static outputSchema() {
|
|
2149
|
-
return outputSchema3;
|
|
2150
|
-
}
|
|
2151
|
-
async execute(input, context) {
|
|
2152
|
-
const {
|
|
2153
|
-
knowledgeBase,
|
|
2154
|
-
queryVector,
|
|
2155
|
-
queryText,
|
|
2156
|
-
topK = 10,
|
|
2157
|
-
filter,
|
|
2158
|
-
scoreThreshold = 0,
|
|
2159
|
-
vectorWeight = 0.7,
|
|
2160
|
-
returnVectors = false
|
|
2161
|
-
} = input;
|
|
2162
|
-
const kb = knowledgeBase;
|
|
2163
1887
|
const searchVector = queryVector instanceof Float32Array ? queryVector : new Float32Array(queryVector);
|
|
2164
|
-
const results = await kb.hybridSearch(searchVector, {
|
|
1888
|
+
const results = method === "hybrid" ? await kb.hybridSearch(searchVector, {
|
|
2165
1889
|
textQuery: queryText,
|
|
2166
1890
|
topK,
|
|
2167
1891
|
filter,
|
|
2168
1892
|
scoreThreshold,
|
|
2169
1893
|
vectorWeight
|
|
1894
|
+
}) : await kb.similaritySearch(searchVector, {
|
|
1895
|
+
topK,
|
|
1896
|
+
filter,
|
|
1897
|
+
scoreThreshold
|
|
2170
1898
|
});
|
|
2171
1899
|
const chunks = results.map((r) => {
|
|
2172
1900
|
const meta = r.metadata;
|
|
@@ -2178,7 +1906,7 @@ class ChunkVectorHybridSearchTask extends Task4 {
|
|
|
2178
1906
|
metadata: results.map((r) => r.metadata),
|
|
2179
1907
|
scores: results.map((r) => r.score),
|
|
2180
1908
|
count: results.length,
|
|
2181
|
-
query
|
|
1909
|
+
query
|
|
2182
1910
|
};
|
|
2183
1911
|
if (returnVectors) {
|
|
2184
1912
|
output.vectors = results.map((r) => r.vector);
|
|
@@ -2186,163 +1914,39 @@ class ChunkVectorHybridSearchTask extends Task4 {
|
|
|
2186
1914
|
return output;
|
|
2187
1915
|
}
|
|
2188
1916
|
}
|
|
2189
|
-
var
|
|
2190
|
-
return new
|
|
2191
|
-
};
|
|
2192
|
-
Workflow5.prototype.hybridSearch = CreateWorkflow5(ChunkVectorHybridSearchTask);
|
|
2193
|
-
|
|
2194
|
-
// src/task/ChunkVectorSearchTask.ts
|
|
2195
|
-
import { TypeKnowledgeBase as TypeKnowledgeBase3 } from "@workglow/knowledge-base";
|
|
2196
|
-
import { CreateWorkflow as CreateWorkflow6, Task as Task5, Workflow as Workflow6 } from "@workglow/task-graph";
|
|
2197
|
-
import {
|
|
2198
|
-
TypedArraySchema as TypedArraySchema5
|
|
2199
|
-
} from "@workglow/util/schema";
|
|
2200
|
-
var inputSchema4 = {
|
|
2201
|
-
type: "object",
|
|
2202
|
-
properties: {
|
|
2203
|
-
knowledgeBase: TypeKnowledgeBase3({
|
|
2204
|
-
title: "Knowledge Base",
|
|
2205
|
-
description: "The knowledge base instance to search in"
|
|
2206
|
-
}),
|
|
2207
|
-
query: TypedArraySchema5({
|
|
2208
|
-
title: "Query Vector",
|
|
2209
|
-
description: "The query vector to search for similar vectors"
|
|
2210
|
-
}),
|
|
2211
|
-
topK: {
|
|
2212
|
-
type: "number",
|
|
2213
|
-
title: "Top K",
|
|
2214
|
-
description: "Number of top results to return",
|
|
2215
|
-
minimum: 1,
|
|
2216
|
-
default: 10
|
|
2217
|
-
},
|
|
2218
|
-
filter: {
|
|
2219
|
-
type: "object",
|
|
2220
|
-
title: "Metadata Filter",
|
|
2221
|
-
description: "Filter results by metadata fields"
|
|
2222
|
-
},
|
|
2223
|
-
scoreThreshold: {
|
|
2224
|
-
type: "number",
|
|
2225
|
-
title: "Score Threshold",
|
|
2226
|
-
description: "Minimum similarity score threshold (0-1)",
|
|
2227
|
-
minimum: 0,
|
|
2228
|
-
maximum: 1,
|
|
2229
|
-
default: 0
|
|
2230
|
-
}
|
|
2231
|
-
},
|
|
2232
|
-
required: ["knowledgeBase", "query"],
|
|
2233
|
-
additionalProperties: false
|
|
2234
|
-
};
|
|
2235
|
-
var outputSchema4 = {
|
|
2236
|
-
type: "object",
|
|
2237
|
-
properties: {
|
|
2238
|
-
ids: {
|
|
2239
|
-
type: "array",
|
|
2240
|
-
items: { type: "string" },
|
|
2241
|
-
title: "IDs",
|
|
2242
|
-
description: "IDs of matching vectors"
|
|
2243
|
-
},
|
|
2244
|
-
vectors: {
|
|
2245
|
-
type: "array",
|
|
2246
|
-
items: TypedArraySchema5({
|
|
2247
|
-
title: "Vector",
|
|
2248
|
-
description: "Matching vector embedding"
|
|
2249
|
-
}),
|
|
2250
|
-
title: "Vectors",
|
|
2251
|
-
description: "Matching vector embeddings"
|
|
2252
|
-
},
|
|
2253
|
-
metadata: {
|
|
2254
|
-
type: "array",
|
|
2255
|
-
items: {
|
|
2256
|
-
type: "object",
|
|
2257
|
-
title: "Metadata",
|
|
2258
|
-
description: "Metadata of matching vector"
|
|
2259
|
-
},
|
|
2260
|
-
title: "Metadata",
|
|
2261
|
-
description: "Metadata of matching vectors"
|
|
2262
|
-
},
|
|
2263
|
-
scores: {
|
|
2264
|
-
type: "array",
|
|
2265
|
-
items: { type: "number" },
|
|
2266
|
-
title: "Scores",
|
|
2267
|
-
description: "Similarity scores for each result"
|
|
2268
|
-
},
|
|
2269
|
-
count: {
|
|
2270
|
-
type: "number",
|
|
2271
|
-
title: "Count",
|
|
2272
|
-
description: "Number of results returned"
|
|
2273
|
-
}
|
|
2274
|
-
},
|
|
2275
|
-
required: ["ids", "vectors", "metadata", "scores", "count"],
|
|
2276
|
-
additionalProperties: false
|
|
2277
|
-
};
|
|
2278
|
-
|
|
2279
|
-
class ChunkVectorSearchTask extends Task5 {
|
|
2280
|
-
static type = "ChunkVectorSearchTask";
|
|
2281
|
-
static category = "Vector Store";
|
|
2282
|
-
static title = "Vector Store Search";
|
|
2283
|
-
static description = "Search for similar vectors in a knowledge base";
|
|
2284
|
-
static cacheable = true;
|
|
2285
|
-
static inputSchema() {
|
|
2286
|
-
return inputSchema4;
|
|
2287
|
-
}
|
|
2288
|
-
static outputSchema() {
|
|
2289
|
-
return outputSchema4;
|
|
2290
|
-
}
|
|
2291
|
-
async execute(input, _context) {
|
|
2292
|
-
const { knowledgeBase, query, topK = 10, filter, scoreThreshold = 0 } = input;
|
|
2293
|
-
const kb = knowledgeBase;
|
|
2294
|
-
const results = await kb.similaritySearch(query, {
|
|
2295
|
-
topK,
|
|
2296
|
-
filter,
|
|
2297
|
-
scoreThreshold
|
|
2298
|
-
});
|
|
2299
|
-
return {
|
|
2300
|
-
ids: results.map((r) => r.chunk_id),
|
|
2301
|
-
vectors: results.map((r) => r.vector),
|
|
2302
|
-
metadata: results.map((r) => r.metadata),
|
|
2303
|
-
scores: results.map((r) => r.score),
|
|
2304
|
-
count: results.length
|
|
2305
|
-
};
|
|
2306
|
-
}
|
|
2307
|
-
}
|
|
2308
|
-
var vectorStoreSearch = (input, config) => {
|
|
2309
|
-
return new ChunkVectorSearchTask(config).run(input);
|
|
1917
|
+
var chunkRetrieval = (input, config) => {
|
|
1918
|
+
return new ChunkRetrievalTask(config).run(input);
|
|
2310
1919
|
};
|
|
2311
|
-
|
|
1920
|
+
Workflow3.prototype.chunkRetrieval = CreateWorkflow3(ChunkRetrievalTask);
|
|
2312
1921
|
|
|
2313
1922
|
// src/task/ChunkVectorUpsertTask.ts
|
|
2314
|
-
import { TypeKnowledgeBase as
|
|
2315
|
-
import { CreateWorkflow as
|
|
1923
|
+
import { ChunkRecordArraySchema, TypeKnowledgeBase as TypeKnowledgeBase2 } from "@workglow/knowledge-base";
|
|
1924
|
+
import { CreateWorkflow as CreateWorkflow4, Task as Task3, Workflow as Workflow4 } from "@workglow/task-graph";
|
|
2316
1925
|
import {
|
|
2317
|
-
TypedArraySchema as
|
|
1926
|
+
TypedArraySchema as TypedArraySchema3
|
|
2318
1927
|
} from "@workglow/util/schema";
|
|
2319
|
-
var
|
|
1928
|
+
var inputSchema2 = {
|
|
2320
1929
|
type: "object",
|
|
2321
1930
|
properties: {
|
|
2322
|
-
knowledgeBase:
|
|
1931
|
+
knowledgeBase: TypeKnowledgeBase2({
|
|
2323
1932
|
title: "Knowledge Base",
|
|
2324
1933
|
description: "The knowledge base instance to store vectors in"
|
|
2325
1934
|
}),
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
title: "Document ID",
|
|
2329
|
-
description: "The document ID"
|
|
2330
|
-
},
|
|
2331
|
-
vectors: TypeSingleOrArray(TypedArraySchema6({
|
|
1935
|
+
chunks: ChunkRecordArraySchema,
|
|
1936
|
+
vector: TypeSingleOrArray(TypedArraySchema3({
|
|
2332
1937
|
title: "Vectors",
|
|
2333
|
-
description: "The vector embeddings"
|
|
1938
|
+
description: "The vector embeddings, aligned 1:1 with chunks"
|
|
2334
1939
|
})),
|
|
2335
|
-
|
|
2336
|
-
type: "
|
|
2337
|
-
title: "
|
|
2338
|
-
description: "
|
|
2339
|
-
|
|
2340
|
-
})
|
|
1940
|
+
doc_title: {
|
|
1941
|
+
type: "string",
|
|
1942
|
+
title: "Document Title",
|
|
1943
|
+
description: "Optional human-readable title stamped onto each chunk's metadata"
|
|
1944
|
+
}
|
|
2341
1945
|
},
|
|
2342
|
-
required: ["knowledgeBase", "
|
|
1946
|
+
required: ["knowledgeBase", "chunks", "vector"],
|
|
2343
1947
|
additionalProperties: false
|
|
2344
1948
|
};
|
|
2345
|
-
var
|
|
1949
|
+
var outputSchema2 = {
|
|
2346
1950
|
type: "object",
|
|
2347
1951
|
properties: {
|
|
2348
1952
|
count: {
|
|
@@ -2353,7 +1957,7 @@ var outputSchema5 = {
|
|
|
2353
1957
|
doc_id: {
|
|
2354
1958
|
type: "string",
|
|
2355
1959
|
title: "Document ID",
|
|
2356
|
-
description: "The document ID"
|
|
1960
|
+
description: "The document ID (read from the first chunk)"
|
|
2357
1961
|
},
|
|
2358
1962
|
chunk_ids: {
|
|
2359
1963
|
type: "array",
|
|
@@ -2366,74 +1970,73 @@ var outputSchema5 = {
|
|
|
2366
1970
|
additionalProperties: false
|
|
2367
1971
|
};
|
|
2368
1972
|
|
|
2369
|
-
class ChunkVectorUpsertTask extends
|
|
1973
|
+
class ChunkVectorUpsertTask extends Task3 {
|
|
2370
1974
|
static type = "ChunkVectorUpsertTask";
|
|
2371
1975
|
static category = "Vector Store";
|
|
2372
1976
|
static title = "Add to Vector Store";
|
|
2373
|
-
static description = "Store
|
|
1977
|
+
static description = "Store chunks + their embeddings in a knowledge base (1:1 aligned)";
|
|
2374
1978
|
static cacheable = false;
|
|
2375
1979
|
static inputSchema() {
|
|
2376
|
-
return
|
|
1980
|
+
return inputSchema2;
|
|
2377
1981
|
}
|
|
2378
1982
|
static outputSchema() {
|
|
2379
|
-
return
|
|
1983
|
+
return outputSchema2;
|
|
2380
1984
|
}
|
|
2381
1985
|
async execute(input, context) {
|
|
2382
|
-
const { knowledgeBase,
|
|
2383
|
-
const
|
|
2384
|
-
const
|
|
1986
|
+
const { knowledgeBase, chunks, vector, doc_title } = input;
|
|
1987
|
+
const chunkArray = chunks;
|
|
1988
|
+
const vectorArray = Array.isArray(vector) ? vector : [vector];
|
|
1989
|
+
if (chunkArray.length !== vectorArray.length) {
|
|
1990
|
+
throw new Error(`Mismatch: ${chunkArray.length} chunks but ${vectorArray.length} vectors — they must be 1:1 aligned`);
|
|
1991
|
+
}
|
|
1992
|
+
if (chunkArray.length === 0) {
|
|
1993
|
+
return { doc_id: "", chunk_ids: [], count: 0 };
|
|
1994
|
+
}
|
|
2385
1995
|
const kb = knowledgeBase;
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
if (
|
|
2389
|
-
throw new Error(
|
|
1996
|
+
const doc_id = chunkArray[0].doc_id;
|
|
1997
|
+
for (let i = 1;i < chunkArray.length; i++) {
|
|
1998
|
+
if (chunkArray[i].doc_id !== doc_id) {
|
|
1999
|
+
throw new Error(`ChunkVectorUpsertTask expects all chunks to share one doc_id, but chunk[${i}] has doc_id=${JSON.stringify(chunkArray[i].doc_id)} (expected ${JSON.stringify(doc_id)}).`);
|
|
2390
2000
|
}
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
}
|
|
2398
|
-
|
|
2399
|
-
const results = await kb.upsertChunksBulk(entities);
|
|
2400
|
-
const chunk_ids = results.map((r) => r.chunk_id);
|
|
2401
|
-
return {
|
|
2402
|
-
doc_id,
|
|
2403
|
-
chunk_ids,
|
|
2404
|
-
count: chunk_ids.length
|
|
2001
|
+
}
|
|
2002
|
+
await context.updateProgress(1, "Upserting vectors");
|
|
2003
|
+
const entities = chunkArray.map((chunk, i) => {
|
|
2004
|
+
const leafNodeId = chunk.leafNodeId ?? chunk.nodePath[chunk.nodePath.length - 1] ?? undefined;
|
|
2005
|
+
const metadata = {
|
|
2006
|
+
...chunk,
|
|
2007
|
+
...leafNodeId !== undefined ? { leafNodeId } : {},
|
|
2008
|
+
...doc_title ? { doc_title } : {}
|
|
2405
2009
|
};
|
|
2406
|
-
} else if (vectorArray.length === 1) {
|
|
2407
|
-
const metadataItem = metadataArray[0];
|
|
2408
|
-
const result = await kb.upsertChunk({
|
|
2409
|
-
doc_id,
|
|
2410
|
-
vector: vectorArray[0],
|
|
2411
|
-
metadata: metadataItem
|
|
2412
|
-
});
|
|
2413
2010
|
return {
|
|
2414
|
-
doc_id,
|
|
2415
|
-
|
|
2416
|
-
|
|
2011
|
+
doc_id: chunk.doc_id,
|
|
2012
|
+
vector: vectorArray[i],
|
|
2013
|
+
metadata
|
|
2417
2014
|
};
|
|
2418
|
-
}
|
|
2015
|
+
});
|
|
2016
|
+
const results = await kb.upsertChunksBulk(entities);
|
|
2017
|
+
const chunk_ids = results.map((r) => r.chunk_id);
|
|
2419
2018
|
return {
|
|
2420
2019
|
doc_id,
|
|
2421
|
-
chunk_ids
|
|
2422
|
-
count:
|
|
2020
|
+
chunk_ids,
|
|
2021
|
+
count: chunk_ids.length
|
|
2423
2022
|
};
|
|
2424
2023
|
}
|
|
2425
2024
|
}
|
|
2426
2025
|
var chunkVectorUpsert = (input, config) => {
|
|
2427
2026
|
return new ChunkVectorUpsertTask(config).run(input);
|
|
2428
2027
|
};
|
|
2429
|
-
|
|
2028
|
+
Workflow4.prototype.chunkVectorUpsert = CreateWorkflow4(ChunkVectorUpsertTask);
|
|
2430
2029
|
|
|
2431
2030
|
// src/task/ContextBuilderTask.ts
|
|
2432
2031
|
import { estimateTokens } from "@workglow/knowledge-base";
|
|
2433
|
-
import {
|
|
2032
|
+
import {
|
|
2033
|
+
CreateWorkflow as CreateWorkflow6,
|
|
2034
|
+
Task as Task4,
|
|
2035
|
+
Workflow as Workflow6
|
|
2036
|
+
} from "@workglow/task-graph";
|
|
2434
2037
|
|
|
2435
2038
|
// src/task/CountTokensTask.ts
|
|
2436
|
-
import { CreateWorkflow as
|
|
2039
|
+
import { CreateWorkflow as CreateWorkflow5, Workflow as Workflow5 } from "@workglow/task-graph";
|
|
2437
2040
|
var modelSchema4 = TypeModel("model");
|
|
2438
2041
|
var CountTokensInputSchema = {
|
|
2439
2042
|
type: "object",
|
|
@@ -2477,7 +2080,7 @@ class CountTokensTask extends AiTask {
|
|
|
2477
2080
|
var countTokens = async (input, config) => {
|
|
2478
2081
|
return new CountTokensTask(config).run(input);
|
|
2479
2082
|
};
|
|
2480
|
-
|
|
2083
|
+
Workflow5.prototype.countTokens = CreateWorkflow5(CountTokensTask);
|
|
2481
2084
|
|
|
2482
2085
|
// src/task/ContextBuilderTask.ts
|
|
2483
2086
|
var ContextFormat = {
|
|
@@ -2491,7 +2094,7 @@ var modelSchema5 = TypeModel("model", {
|
|
|
2491
2094
|
title: "Model",
|
|
2492
2095
|
description: "Model to use for token counting (optional, falls back to estimation)"
|
|
2493
2096
|
});
|
|
2494
|
-
var
|
|
2097
|
+
var inputSchema3 = {
|
|
2495
2098
|
type: "object",
|
|
2496
2099
|
properties: {
|
|
2497
2100
|
chunks: {
|
|
@@ -2556,7 +2159,7 @@ var inputSchema6 = {
|
|
|
2556
2159
|
required: ["chunks"],
|
|
2557
2160
|
additionalProperties: false
|
|
2558
2161
|
};
|
|
2559
|
-
var
|
|
2162
|
+
var outputSchema3 = {
|
|
2560
2163
|
type: "object",
|
|
2561
2164
|
properties: {
|
|
2562
2165
|
context: {
|
|
@@ -2584,19 +2187,22 @@ var outputSchema6 = {
|
|
|
2584
2187
|
additionalProperties: false
|
|
2585
2188
|
};
|
|
2586
2189
|
|
|
2587
|
-
class ContextBuilderTask extends
|
|
2190
|
+
class ContextBuilderTask extends Task4 {
|
|
2588
2191
|
static type = "ContextBuilderTask";
|
|
2589
2192
|
static category = "RAG";
|
|
2590
2193
|
static title = "Context Builder";
|
|
2591
2194
|
static description = "Format retrieved chunks into context for LLM prompts";
|
|
2592
2195
|
static cacheable = true;
|
|
2593
2196
|
static inputSchema() {
|
|
2594
|
-
return
|
|
2197
|
+
return inputSchema3;
|
|
2595
2198
|
}
|
|
2596
2199
|
static outputSchema() {
|
|
2597
|
-
return
|
|
2200
|
+
return outputSchema3;
|
|
2201
|
+
}
|
|
2202
|
+
async execute(input, context) {
|
|
2203
|
+
return this.executePreview(input, context);
|
|
2598
2204
|
}
|
|
2599
|
-
async
|
|
2205
|
+
async executePreview(input, context) {
|
|
2600
2206
|
const {
|
|
2601
2207
|
chunks,
|
|
2602
2208
|
metadata = [],
|
|
@@ -2777,14 +2383,14 @@ class ContextBuilderTask extends Task7 {
|
|
|
2777
2383
|
var contextBuilder = (input, config) => {
|
|
2778
2384
|
return new ContextBuilderTask(config).run(input);
|
|
2779
2385
|
};
|
|
2780
|
-
|
|
2386
|
+
Workflow6.prototype.contextBuilder = CreateWorkflow6(ContextBuilderTask);
|
|
2781
2387
|
|
|
2782
2388
|
// src/task/DocumentEnricherTask.ts
|
|
2783
2389
|
import { getChildren, hasChildren } from "@workglow/knowledge-base";
|
|
2784
|
-
import { CreateWorkflow as
|
|
2390
|
+
import { CreateWorkflow as CreateWorkflow9, Task as Task5, Workflow as Workflow9 } from "@workglow/task-graph";
|
|
2785
2391
|
|
|
2786
2392
|
// src/task/TextNamedEntityRecognitionTask.ts
|
|
2787
|
-
import { CreateWorkflow as
|
|
2393
|
+
import { CreateWorkflow as CreateWorkflow7, Workflow as Workflow7 } from "@workglow/task-graph";
|
|
2788
2394
|
var modelSchema6 = TypeModel("model:TextNamedEntityRecognitionTask");
|
|
2789
2395
|
var TextNamedEntityRecognitionInputSchema = {
|
|
2790
2396
|
type: "object",
|
|
@@ -2859,10 +2465,10 @@ class TextNamedEntityRecognitionTask extends AiTask {
|
|
|
2859
2465
|
var textNamedEntityRecognition = (input, config) => {
|
|
2860
2466
|
return new TextNamedEntityRecognitionTask(config).run(input);
|
|
2861
2467
|
};
|
|
2862
|
-
|
|
2468
|
+
Workflow7.prototype.textNamedEntityRecognition = CreateWorkflow7(TextNamedEntityRecognitionTask);
|
|
2863
2469
|
|
|
2864
2470
|
// src/task/TextSummaryTask.ts
|
|
2865
|
-
import { CreateWorkflow as
|
|
2471
|
+
import { CreateWorkflow as CreateWorkflow8, Workflow as Workflow8 } from "@workglow/task-graph";
|
|
2866
2472
|
var modelSchema7 = TypeModel("model:TextSummaryTask");
|
|
2867
2473
|
var TextSummaryInputSchema = {
|
|
2868
2474
|
type: "object",
|
|
@@ -2906,10 +2512,10 @@ class TextSummaryTask extends StreamingAiTask {
|
|
|
2906
2512
|
var textSummary = async (input, config) => {
|
|
2907
2513
|
return new TextSummaryTask(config).run(input);
|
|
2908
2514
|
};
|
|
2909
|
-
|
|
2515
|
+
Workflow8.prototype.textSummary = CreateWorkflow8(TextSummaryTask);
|
|
2910
2516
|
|
|
2911
2517
|
// src/task/DocumentEnricherTask.ts
|
|
2912
|
-
var
|
|
2518
|
+
var inputSchema4 = {
|
|
2913
2519
|
type: "object",
|
|
2914
2520
|
properties: {
|
|
2915
2521
|
doc_id: {
|
|
@@ -2953,7 +2559,7 @@ var inputSchema7 = {
|
|
|
2953
2559
|
required: [],
|
|
2954
2560
|
additionalProperties: false
|
|
2955
2561
|
};
|
|
2956
|
-
var
|
|
2562
|
+
var outputSchema4 = {
|
|
2957
2563
|
type: "object",
|
|
2958
2564
|
properties: {
|
|
2959
2565
|
doc_id: {
|
|
@@ -2980,17 +2586,17 @@ var outputSchema7 = {
|
|
|
2980
2586
|
additionalProperties: false
|
|
2981
2587
|
};
|
|
2982
2588
|
|
|
2983
|
-
class DocumentEnricherTask extends
|
|
2589
|
+
class DocumentEnricherTask extends Task5 {
|
|
2984
2590
|
static type = "DocumentEnricherTask";
|
|
2985
2591
|
static category = "Document";
|
|
2986
2592
|
static title = "Document Enricher";
|
|
2987
2593
|
static description = "Enrich document nodes with summaries and entities";
|
|
2988
2594
|
static cacheable = true;
|
|
2989
2595
|
static inputSchema() {
|
|
2990
|
-
return
|
|
2596
|
+
return inputSchema4;
|
|
2991
2597
|
}
|
|
2992
2598
|
static outputSchema() {
|
|
2993
|
-
return
|
|
2599
|
+
return outputSchema4;
|
|
2994
2600
|
}
|
|
2995
2601
|
async execute(input, context) {
|
|
2996
2602
|
const {
|
|
@@ -3139,16 +2745,16 @@ class DocumentEnricherTask extends Task8 {
|
|
|
3139
2745
|
var documentEnricher = (input, config) => {
|
|
3140
2746
|
return new DocumentEnricherTask(config).run(input);
|
|
3141
2747
|
};
|
|
3142
|
-
|
|
2748
|
+
Workflow9.prototype.documentEnricher = CreateWorkflow9(DocumentEnricherTask);
|
|
3143
2749
|
|
|
3144
2750
|
// src/task/DocumentUpsertTask.ts
|
|
3145
|
-
import { Document, TypeKnowledgeBase as
|
|
2751
|
+
import { Document, TypeKnowledgeBase as TypeKnowledgeBase3 } from "@workglow/knowledge-base";
|
|
3146
2752
|
import { DocumentMetadataSchema } from "@workglow/knowledge-base";
|
|
3147
|
-
import { CreateWorkflow as
|
|
3148
|
-
var
|
|
2753
|
+
import { CreateWorkflow as CreateWorkflow10, Task as Task6, Workflow as Workflow10 } from "@workglow/task-graph";
|
|
2754
|
+
var inputSchema5 = {
|
|
3149
2755
|
type: "object",
|
|
3150
2756
|
properties: {
|
|
3151
|
-
knowledgeBase:
|
|
2757
|
+
knowledgeBase: TypeKnowledgeBase3({
|
|
3152
2758
|
title: "Knowledge Base",
|
|
3153
2759
|
description: "The knowledge base instance to store the document in"
|
|
3154
2760
|
}),
|
|
@@ -3176,7 +2782,7 @@ var inputSchema8 = {
|
|
|
3176
2782
|
required: ["knowledgeBase", "doc_id", "documentTree"],
|
|
3177
2783
|
additionalProperties: false
|
|
3178
2784
|
};
|
|
3179
|
-
var
|
|
2785
|
+
var outputSchema5 = {
|
|
3180
2786
|
type: "object",
|
|
3181
2787
|
properties: {
|
|
3182
2788
|
doc_id: {
|
|
@@ -3189,17 +2795,17 @@ var outputSchema8 = {
|
|
|
3189
2795
|
additionalProperties: false
|
|
3190
2796
|
};
|
|
3191
2797
|
|
|
3192
|
-
class DocumentUpsertTask extends
|
|
2798
|
+
class DocumentUpsertTask extends Task6 {
|
|
3193
2799
|
static type = "DocumentUpsertTask";
|
|
3194
2800
|
static category = "Vector Store";
|
|
3195
2801
|
static title = "Add Document";
|
|
3196
2802
|
static description = "Persist a parsed document tree to a knowledge base";
|
|
3197
2803
|
static cacheable = false;
|
|
3198
2804
|
static inputSchema() {
|
|
3199
|
-
return
|
|
2805
|
+
return inputSchema5;
|
|
3200
2806
|
}
|
|
3201
2807
|
static outputSchema() {
|
|
3202
|
-
return
|
|
2808
|
+
return outputSchema5;
|
|
3203
2809
|
}
|
|
3204
2810
|
async execute(input, context) {
|
|
3205
2811
|
const { knowledgeBase, doc_id, documentTree, title, metadata } = input;
|
|
@@ -3222,10 +2828,10 @@ class DocumentUpsertTask extends Task9 {
|
|
|
3222
2828
|
var documentUpsert = (input, config) => {
|
|
3223
2829
|
return new DocumentUpsertTask(config).run(input);
|
|
3224
2830
|
};
|
|
3225
|
-
|
|
2831
|
+
Workflow10.prototype.documentUpsert = CreateWorkflow10(DocumentUpsertTask);
|
|
3226
2832
|
|
|
3227
2833
|
// src/task/DownloadModelTask.ts
|
|
3228
|
-
import { CreateWorkflow as
|
|
2834
|
+
import { CreateWorkflow as CreateWorkflow11, Workflow as Workflow11 } from "@workglow/task-graph";
|
|
3229
2835
|
var modelSchema8 = TypeModel("model");
|
|
3230
2836
|
var DownloadModelInputSchema = {
|
|
3231
2837
|
type: "object",
|
|
@@ -3292,10 +2898,10 @@ class DownloadModelTask extends AiTask {
|
|
|
3292
2898
|
var downloadModel = (input, config) => {
|
|
3293
2899
|
return new DownloadModelTask(config).run(input);
|
|
3294
2900
|
};
|
|
3295
|
-
|
|
2901
|
+
Workflow11.prototype.downloadModel = CreateWorkflow11(DownloadModelTask);
|
|
3296
2902
|
|
|
3297
2903
|
// src/task/FaceDetectorTask.ts
|
|
3298
|
-
import { CreateWorkflow as
|
|
2904
|
+
import { CreateWorkflow as CreateWorkflow12, Workflow as Workflow12 } from "@workglow/task-graph";
|
|
3299
2905
|
var modelSchema9 = TypeModel("model:FaceDetectorTask");
|
|
3300
2906
|
var TypeBoundingBox2 = {
|
|
3301
2907
|
type: "object",
|
|
@@ -3423,10 +3029,10 @@ class FaceDetectorTask extends AiVisionTask {
|
|
|
3423
3029
|
var faceDetector = (input, config) => {
|
|
3424
3030
|
return new FaceDetectorTask(config).run(input);
|
|
3425
3031
|
};
|
|
3426
|
-
|
|
3032
|
+
Workflow12.prototype.faceDetector = CreateWorkflow12(FaceDetectorTask);
|
|
3427
3033
|
|
|
3428
3034
|
// src/task/FaceLandmarkerTask.ts
|
|
3429
|
-
import { CreateWorkflow as
|
|
3035
|
+
import { CreateWorkflow as CreateWorkflow13, Workflow as Workflow13 } from "@workglow/task-graph";
|
|
3430
3036
|
var modelSchema10 = TypeModel("model:FaceLandmarkerTask");
|
|
3431
3037
|
var TypeLandmark = {
|
|
3432
3038
|
type: "object",
|
|
@@ -3585,10 +3191,10 @@ class FaceLandmarkerTask extends AiVisionTask {
|
|
|
3585
3191
|
var faceLandmarker = (input, config) => {
|
|
3586
3192
|
return new FaceLandmarkerTask(config).run(input);
|
|
3587
3193
|
};
|
|
3588
|
-
|
|
3194
|
+
Workflow13.prototype.faceLandmarker = CreateWorkflow13(FaceLandmarkerTask);
|
|
3589
3195
|
|
|
3590
3196
|
// src/task/GestureRecognizerTask.ts
|
|
3591
|
-
import { CreateWorkflow as
|
|
3197
|
+
import { CreateWorkflow as CreateWorkflow14, Workflow as Workflow14 } from "@workglow/task-graph";
|
|
3592
3198
|
var modelSchema11 = TypeModel("model:GestureRecognizerTask");
|
|
3593
3199
|
var TypeLandmark2 = {
|
|
3594
3200
|
type: "object",
|
|
@@ -3753,10 +3359,10 @@ class GestureRecognizerTask extends AiVisionTask {
|
|
|
3753
3359
|
var gestureRecognizer = (input, config) => {
|
|
3754
3360
|
return new GestureRecognizerTask(config).run(input);
|
|
3755
3361
|
};
|
|
3756
|
-
|
|
3362
|
+
Workflow14.prototype.gestureRecognizer = CreateWorkflow14(GestureRecognizerTask);
|
|
3757
3363
|
|
|
3758
3364
|
// src/task/HandLandmarkerTask.ts
|
|
3759
|
-
import { CreateWorkflow as
|
|
3365
|
+
import { CreateWorkflow as CreateWorkflow15, Workflow as Workflow15 } from "@workglow/task-graph";
|
|
3760
3366
|
var modelSchema12 = TypeModel("model:HandLandmarkerTask");
|
|
3761
3367
|
var TypeLandmark3 = {
|
|
3762
3368
|
type: "object",
|
|
@@ -3898,22 +3504,22 @@ class HandLandmarkerTask extends AiVisionTask {
|
|
|
3898
3504
|
var handLandmarker = (input, config) => {
|
|
3899
3505
|
return new HandLandmarkerTask(config).run(input);
|
|
3900
3506
|
};
|
|
3901
|
-
|
|
3507
|
+
Workflow15.prototype.handLandmarker = CreateWorkflow15(HandLandmarkerTask);
|
|
3902
3508
|
|
|
3903
3509
|
// src/task/HierarchicalChunkerTask.ts
|
|
3904
3510
|
import {
|
|
3905
|
-
ChunkRecordSchema
|
|
3511
|
+
ChunkRecordSchema,
|
|
3906
3512
|
estimateTokens as estimateTokens2,
|
|
3907
3513
|
getChildren as getChildren2,
|
|
3908
3514
|
hasChildren as hasChildren2
|
|
3909
3515
|
} from "@workglow/knowledge-base";
|
|
3910
|
-
import { CreateWorkflow as
|
|
3516
|
+
import { CreateWorkflow as CreateWorkflow16, Task as Task7, Workflow as Workflow16 } from "@workglow/task-graph";
|
|
3911
3517
|
import { uuid4 } from "@workglow/util";
|
|
3912
3518
|
var modelSchema13 = TypeModel("model", {
|
|
3913
3519
|
title: "Model",
|
|
3914
3520
|
description: "Model to use for token counting"
|
|
3915
3521
|
});
|
|
3916
|
-
var
|
|
3522
|
+
var inputSchema6 = {
|
|
3917
3523
|
type: "object",
|
|
3918
3524
|
properties: {
|
|
3919
3525
|
doc_id: {
|
|
@@ -3960,7 +3566,7 @@ var inputSchema9 = {
|
|
|
3960
3566
|
required: ["doc_id", "documentTree"],
|
|
3961
3567
|
additionalProperties: false
|
|
3962
3568
|
};
|
|
3963
|
-
var
|
|
3569
|
+
var outputSchema6 = {
|
|
3964
3570
|
type: "object",
|
|
3965
3571
|
properties: {
|
|
3966
3572
|
doc_id: {
|
|
@@ -3970,7 +3576,7 @@ var outputSchema9 = {
|
|
|
3970
3576
|
},
|
|
3971
3577
|
chunks: {
|
|
3972
3578
|
type: "array",
|
|
3973
|
-
items:
|
|
3579
|
+
items: ChunkRecordSchema(),
|
|
3974
3580
|
title: "Chunks",
|
|
3975
3581
|
description: "Array of chunk records"
|
|
3976
3582
|
},
|
|
@@ -3990,17 +3596,17 @@ var outputSchema9 = {
|
|
|
3990
3596
|
additionalProperties: false
|
|
3991
3597
|
};
|
|
3992
3598
|
|
|
3993
|
-
class HierarchicalChunkerTask extends
|
|
3599
|
+
class HierarchicalChunkerTask extends Task7 {
|
|
3994
3600
|
static type = "HierarchicalChunkerTask";
|
|
3995
3601
|
static category = "Document";
|
|
3996
3602
|
static title = "Hierarchical Chunker";
|
|
3997
3603
|
static description = "Chunk documents hierarchically respecting token budgets";
|
|
3998
3604
|
static cacheable = true;
|
|
3999
3605
|
static inputSchema() {
|
|
4000
|
-
return
|
|
3606
|
+
return inputSchema6;
|
|
4001
3607
|
}
|
|
4002
3608
|
static outputSchema() {
|
|
4003
|
-
return
|
|
3609
|
+
return outputSchema6;
|
|
4004
3610
|
}
|
|
4005
3611
|
async execute(input, context) {
|
|
4006
3612
|
const {
|
|
@@ -4134,36 +3740,36 @@ class HierarchicalChunkerTask extends Task10 {
|
|
|
4134
3740
|
var hierarchicalChunker = (input, config) => {
|
|
4135
3741
|
return new HierarchicalChunkerTask(config).run(input);
|
|
4136
3742
|
};
|
|
4137
|
-
|
|
3743
|
+
Workflow16.prototype.hierarchicalChunker = CreateWorkflow16(HierarchicalChunkerTask);
|
|
4138
3744
|
|
|
4139
3745
|
// src/task/HierarchyJoinTask.ts
|
|
4140
|
-
import { ChunkRecordArraySchema, TypeKnowledgeBase as
|
|
4141
|
-
import { CreateWorkflow as
|
|
4142
|
-
var
|
|
3746
|
+
import { ChunkRecordArraySchema as ChunkRecordArraySchema2, TypeKnowledgeBase as TypeKnowledgeBase4 } from "@workglow/knowledge-base";
|
|
3747
|
+
import { CreateWorkflow as CreateWorkflow17, Task as Task8, Workflow as Workflow17 } from "@workglow/task-graph";
|
|
3748
|
+
var inputSchema7 = {
|
|
4143
3749
|
type: "object",
|
|
4144
3750
|
properties: {
|
|
4145
|
-
knowledgeBase:
|
|
3751
|
+
knowledgeBase: TypeKnowledgeBase4({
|
|
4146
3752
|
title: "Knowledge Base",
|
|
4147
3753
|
description: "The knowledge base to query for hierarchy"
|
|
4148
3754
|
}),
|
|
3755
|
+
metadata: ChunkRecordArraySchema2,
|
|
4149
3756
|
chunks: {
|
|
4150
3757
|
type: "array",
|
|
4151
3758
|
items: { type: "string" },
|
|
4152
3759
|
title: "Chunks",
|
|
4153
|
-
description: "Retrieved text chunks"
|
|
3760
|
+
description: "Retrieved text chunks (pass-through)"
|
|
4154
3761
|
},
|
|
4155
3762
|
chunk_ids: {
|
|
4156
3763
|
type: "array",
|
|
4157
3764
|
items: { type: "string" },
|
|
4158
3765
|
title: "Chunk IDs",
|
|
4159
|
-
description: "IDs of retrieved chunks"
|
|
3766
|
+
description: "IDs of retrieved chunks (pass-through)"
|
|
4160
3767
|
},
|
|
4161
|
-
metadata: ChunkRecordArraySchema,
|
|
4162
3768
|
scores: {
|
|
4163
3769
|
type: "array",
|
|
4164
3770
|
items: { type: "number" },
|
|
4165
3771
|
title: "Scores",
|
|
4166
|
-
description: "Similarity scores
|
|
3772
|
+
description: "Similarity scores (pass-through)"
|
|
4167
3773
|
},
|
|
4168
3774
|
includeParentSummaries: {
|
|
4169
3775
|
type: "boolean",
|
|
@@ -4178,74 +3784,72 @@ var inputSchema10 = {
|
|
|
4178
3784
|
default: true
|
|
4179
3785
|
}
|
|
4180
3786
|
},
|
|
4181
|
-
required: ["knowledgeBase", "
|
|
3787
|
+
required: ["knowledgeBase", "metadata"],
|
|
4182
3788
|
additionalProperties: false
|
|
4183
3789
|
};
|
|
4184
|
-
var
|
|
3790
|
+
var outputSchema7 = {
|
|
4185
3791
|
type: "object",
|
|
4186
3792
|
properties: {
|
|
3793
|
+
metadata: ChunkRecordArraySchema2,
|
|
4187
3794
|
chunks: {
|
|
4188
3795
|
type: "array",
|
|
4189
3796
|
items: { type: "string" },
|
|
4190
3797
|
title: "Chunks",
|
|
4191
|
-
description: "Retrieved text chunks"
|
|
3798
|
+
description: "Retrieved text chunks (pass-through)"
|
|
4192
3799
|
},
|
|
4193
3800
|
chunk_ids: {
|
|
4194
3801
|
type: "array",
|
|
4195
3802
|
items: { type: "string" },
|
|
4196
3803
|
title: "Chunk IDs",
|
|
4197
|
-
description: "IDs of retrieved chunks"
|
|
3804
|
+
description: "IDs of retrieved chunks (pass-through)"
|
|
4198
3805
|
},
|
|
4199
|
-
metadata: ChunkRecordArraySchema,
|
|
4200
3806
|
scores: {
|
|
4201
3807
|
type: "array",
|
|
4202
3808
|
items: { type: "number" },
|
|
4203
3809
|
title: "Scores",
|
|
4204
|
-
description: "Similarity scores"
|
|
3810
|
+
description: "Similarity scores (pass-through)"
|
|
4205
3811
|
},
|
|
4206
3812
|
count: {
|
|
4207
3813
|
type: "number",
|
|
4208
3814
|
title: "Count",
|
|
4209
|
-
description: "Number of
|
|
3815
|
+
description: "Number of enriched records"
|
|
4210
3816
|
}
|
|
4211
3817
|
},
|
|
4212
|
-
required: ["
|
|
3818
|
+
required: ["metadata", "count"],
|
|
4213
3819
|
additionalProperties: false
|
|
4214
3820
|
};
|
|
4215
3821
|
|
|
4216
|
-
class HierarchyJoinTask extends
|
|
3822
|
+
class HierarchyJoinTask extends Task8 {
|
|
4217
3823
|
static type = "HierarchyJoinTask";
|
|
4218
3824
|
static category = "RAG";
|
|
4219
3825
|
static title = "Hierarchy Join";
|
|
4220
|
-
static description = "Enrich
|
|
3826
|
+
static description = "Enrich retrieval metadata with document hierarchy context";
|
|
4221
3827
|
static cacheable = false;
|
|
4222
3828
|
static inputSchema() {
|
|
4223
|
-
return
|
|
3829
|
+
return inputSchema7;
|
|
4224
3830
|
}
|
|
4225
3831
|
static outputSchema() {
|
|
4226
|
-
return
|
|
3832
|
+
return outputSchema7;
|
|
4227
3833
|
}
|
|
4228
3834
|
async execute(input, context) {
|
|
4229
3835
|
const {
|
|
4230
3836
|
knowledgeBase,
|
|
3837
|
+
metadata,
|
|
4231
3838
|
chunks,
|
|
4232
3839
|
chunk_ids,
|
|
4233
|
-
metadata,
|
|
4234
3840
|
scores,
|
|
4235
3841
|
includeParentSummaries = true,
|
|
4236
3842
|
includeEntities = true
|
|
4237
3843
|
} = input;
|
|
4238
3844
|
const kb = knowledgeBase;
|
|
4239
3845
|
const enrichedMetadata = [];
|
|
4240
|
-
for (
|
|
4241
|
-
const chunkId = chunk_ids[i];
|
|
4242
|
-
const originalMetadata = metadata[i];
|
|
3846
|
+
for (const originalMetadata of metadata) {
|
|
4243
3847
|
if (!originalMetadata) {
|
|
4244
3848
|
enrichedMetadata.push({});
|
|
4245
3849
|
continue;
|
|
4246
3850
|
}
|
|
4247
3851
|
const doc_id = originalMetadata.doc_id;
|
|
4248
|
-
const leafNodeId = originalMetadata.leafNodeId;
|
|
3852
|
+
const leafNodeId = originalMetadata.leafNodeId ?? originalMetadata.nodePath?.[originalMetadata.nodePath.length - 1];
|
|
4249
3853
|
if (!doc_id || !leafNodeId) {
|
|
4250
3854
|
enrichedMetadata.push(originalMetadata);
|
|
4251
3855
|
continue;
|
|
@@ -4291,31 +3895,35 @@ class HierarchyJoinTask extends Task11 {
|
|
|
4291
3895
|
}
|
|
4292
3896
|
enrichedMetadata.push(enriched);
|
|
4293
3897
|
} catch (error) {
|
|
4294
|
-
console.error(`Failed to join hierarchy for chunk ${chunkId}:`, error);
|
|
3898
|
+
console.error(`Failed to join hierarchy for chunk ${originalMetadata.chunkId}:`, error);
|
|
4295
3899
|
enrichedMetadata.push(originalMetadata);
|
|
4296
3900
|
}
|
|
4297
3901
|
}
|
|
4298
|
-
|
|
4299
|
-
chunks,
|
|
4300
|
-
chunk_ids,
|
|
3902
|
+
const output = {
|
|
4301
3903
|
metadata: enrichedMetadata,
|
|
4302
|
-
|
|
4303
|
-
count: chunks.length
|
|
3904
|
+
count: enrichedMetadata.length
|
|
4304
3905
|
};
|
|
3906
|
+
if (chunks !== undefined)
|
|
3907
|
+
output.chunks = chunks;
|
|
3908
|
+
if (chunk_ids !== undefined)
|
|
3909
|
+
output.chunk_ids = chunk_ids;
|
|
3910
|
+
if (scores !== undefined)
|
|
3911
|
+
output.scores = scores;
|
|
3912
|
+
return output;
|
|
4305
3913
|
}
|
|
4306
3914
|
}
|
|
4307
3915
|
var hierarchyJoin = (input, config) => {
|
|
4308
3916
|
return new HierarchyJoinTask(config).run(input);
|
|
4309
3917
|
};
|
|
4310
|
-
|
|
3918
|
+
Workflow17.prototype.hierarchyJoin = CreateWorkflow17(HierarchyJoinTask);
|
|
4311
3919
|
|
|
4312
3920
|
// src/task/KbToDocumentsTask.ts
|
|
4313
|
-
import { TypeKnowledgeBase as
|
|
4314
|
-
import { CreateWorkflow as
|
|
4315
|
-
var
|
|
3921
|
+
import { TypeKnowledgeBase as TypeKnowledgeBase5 } from "@workglow/knowledge-base";
|
|
3922
|
+
import { CreateWorkflow as CreateWorkflow18, Task as Task9, Workflow as Workflow18 } from "@workglow/task-graph";
|
|
3923
|
+
var inputSchema8 = {
|
|
4316
3924
|
type: "object",
|
|
4317
3925
|
properties: {
|
|
4318
|
-
knowledgeBase:
|
|
3926
|
+
knowledgeBase: TypeKnowledgeBase5({
|
|
4319
3927
|
title: "Knowledge Base",
|
|
4320
3928
|
description: "The knowledge base instance to list documents from"
|
|
4321
3929
|
}),
|
|
@@ -4329,7 +3937,7 @@ var inputSchema11 = {
|
|
|
4329
3937
|
required: ["knowledgeBase"],
|
|
4330
3938
|
additionalProperties: false
|
|
4331
3939
|
};
|
|
4332
|
-
var
|
|
3940
|
+
var outputSchema8 = {
|
|
4333
3941
|
type: "object",
|
|
4334
3942
|
properties: {
|
|
4335
3943
|
doc_id: {
|
|
@@ -4355,17 +3963,17 @@ var outputSchema11 = {
|
|
|
4355
3963
|
additionalProperties: false
|
|
4356
3964
|
};
|
|
4357
3965
|
|
|
4358
|
-
class KbToDocumentsTask extends
|
|
3966
|
+
class KbToDocumentsTask extends Task9 {
|
|
4359
3967
|
static type = "KbToDocumentsTask";
|
|
4360
3968
|
static category = "Vector Store";
|
|
4361
3969
|
static title = "Knowledge Base to Documents";
|
|
4362
3970
|
static description = "List documents from a knowledge base, optionally filtering to only those that need embedding";
|
|
4363
3971
|
static cacheable = false;
|
|
4364
3972
|
static inputSchema() {
|
|
4365
|
-
return
|
|
3973
|
+
return inputSchema8;
|
|
4366
3974
|
}
|
|
4367
3975
|
static outputSchema() {
|
|
4368
|
-
return
|
|
3976
|
+
return outputSchema8;
|
|
4369
3977
|
}
|
|
4370
3978
|
async execute(input, context) {
|
|
4371
3979
|
const { knowledgeBase, onlyStale = true } = input;
|
|
@@ -4396,10 +4004,10 @@ class KbToDocumentsTask extends Task12 {
|
|
|
4396
4004
|
var kbToDocuments = (input, config) => {
|
|
4397
4005
|
return new KbToDocumentsTask(config).run(input);
|
|
4398
4006
|
};
|
|
4399
|
-
|
|
4007
|
+
Workflow18.prototype.kbToDocuments = CreateWorkflow18(KbToDocumentsTask);
|
|
4400
4008
|
|
|
4401
4009
|
// src/task/ImageClassificationTask.ts
|
|
4402
|
-
import { CreateWorkflow as
|
|
4010
|
+
import { CreateWorkflow as CreateWorkflow19, Workflow as Workflow19 } from "@workglow/task-graph";
|
|
4403
4011
|
var modelSchema14 = TypeModel("model:ImageClassificationTask");
|
|
4404
4012
|
var ImageClassificationInputSchema = {
|
|
4405
4013
|
type: "object",
|
|
@@ -4459,12 +4067,12 @@ class ImageClassificationTask extends AiVisionTask {
|
|
|
4459
4067
|
var imageClassification = (input, config) => {
|
|
4460
4068
|
return new ImageClassificationTask(config).run(input);
|
|
4461
4069
|
};
|
|
4462
|
-
|
|
4070
|
+
Workflow19.prototype.imageClassification = CreateWorkflow19(ImageClassificationTask);
|
|
4463
4071
|
|
|
4464
4072
|
// src/task/ImageEmbeddingTask.ts
|
|
4465
|
-
import { CreateWorkflow as
|
|
4073
|
+
import { CreateWorkflow as CreateWorkflow20, Workflow as Workflow20 } from "@workglow/task-graph";
|
|
4466
4074
|
import {
|
|
4467
|
-
TypedArraySchema as
|
|
4075
|
+
TypedArraySchema as TypedArraySchema4
|
|
4468
4076
|
} from "@workglow/util/schema";
|
|
4469
4077
|
var modelSchema15 = TypeModel("model:ImageEmbeddingTask");
|
|
4470
4078
|
var ImageEmbeddingInputSchema = {
|
|
@@ -4479,7 +4087,7 @@ var ImageEmbeddingInputSchema = {
|
|
|
4479
4087
|
var ImageEmbeddingOutputSchema = {
|
|
4480
4088
|
type: "object",
|
|
4481
4089
|
properties: {
|
|
4482
|
-
vector: TypeSingleOrArray(
|
|
4090
|
+
vector: TypeSingleOrArray(TypedArraySchema4({
|
|
4483
4091
|
title: "Embedding",
|
|
4484
4092
|
description: "The image embedding vector"
|
|
4485
4093
|
}))
|
|
@@ -4503,10 +4111,10 @@ class ImageEmbeddingTask extends AiVisionTask {
|
|
|
4503
4111
|
var imageEmbedding = (input, config) => {
|
|
4504
4112
|
return new ImageEmbeddingTask(config).run(input);
|
|
4505
4113
|
};
|
|
4506
|
-
|
|
4114
|
+
Workflow20.prototype.imageEmbedding = CreateWorkflow20(ImageEmbeddingTask);
|
|
4507
4115
|
|
|
4508
4116
|
// src/task/ImageSegmentationTask.ts
|
|
4509
|
-
import { CreateWorkflow as
|
|
4117
|
+
import { CreateWorkflow as CreateWorkflow21, Workflow as Workflow21 } from "@workglow/task-graph";
|
|
4510
4118
|
var modelSchema16 = TypeModel("model:ImageSegmentationTask");
|
|
4511
4119
|
var ImageSegmentationInputSchema = {
|
|
4512
4120
|
type: "object",
|
|
@@ -4591,10 +4199,10 @@ class ImageSegmentationTask extends AiVisionTask {
|
|
|
4591
4199
|
var imageSegmentation = (input, config) => {
|
|
4592
4200
|
return new ImageSegmentationTask(config).run(input);
|
|
4593
4201
|
};
|
|
4594
|
-
|
|
4202
|
+
Workflow21.prototype.imageSegmentation = CreateWorkflow21(ImageSegmentationTask);
|
|
4595
4203
|
|
|
4596
4204
|
// src/task/ImageToTextTask.ts
|
|
4597
|
-
import { CreateWorkflow as
|
|
4205
|
+
import { CreateWorkflow as CreateWorkflow22, Workflow as Workflow22 } from "@workglow/task-graph";
|
|
4598
4206
|
var modelSchema17 = TypeModel("model:ImageToTextTask");
|
|
4599
4207
|
var generatedTextSchema = {
|
|
4600
4208
|
type: "string",
|
|
@@ -4646,10 +4254,10 @@ class ImageToTextTask extends AiVisionTask {
|
|
|
4646
4254
|
var imageToText = (input, config) => {
|
|
4647
4255
|
return new ImageToTextTask(config).run(input);
|
|
4648
4256
|
};
|
|
4649
|
-
|
|
4257
|
+
Workflow22.prototype.imageToText = CreateWorkflow22(ImageToTextTask);
|
|
4650
4258
|
|
|
4651
4259
|
// src/task/ModelInfoTask.ts
|
|
4652
|
-
import { CreateWorkflow as
|
|
4260
|
+
import { CreateWorkflow as CreateWorkflow23, Workflow as Workflow23 } from "@workglow/task-graph";
|
|
4653
4261
|
var modelSchema18 = TypeModel("model");
|
|
4654
4262
|
var ModelInfoInputSchema = {
|
|
4655
4263
|
type: "object",
|
|
@@ -4725,10 +4333,10 @@ class ModelInfoTask extends AiTask {
|
|
|
4725
4333
|
var modelInfo = (input, config) => {
|
|
4726
4334
|
return new ModelInfoTask(config).run(input);
|
|
4727
4335
|
};
|
|
4728
|
-
|
|
4336
|
+
Workflow23.prototype.modelInfo = CreateWorkflow23(ModelInfoTask);
|
|
4729
4337
|
|
|
4730
4338
|
// src/task/ModelSearchTask.ts
|
|
4731
|
-
import { CreateWorkflow as
|
|
4339
|
+
import { CreateWorkflow as CreateWorkflow24, Task as Task10, Workflow as Workflow24 } from "@workglow/task-graph";
|
|
4732
4340
|
var ModelSearchInputSchema = {
|
|
4733
4341
|
type: "object",
|
|
4734
4342
|
properties: {
|
|
@@ -4793,7 +4401,7 @@ var ModelSearchOutputSchema = {
|
|
|
4793
4401
|
additionalProperties: false
|
|
4794
4402
|
};
|
|
4795
4403
|
|
|
4796
|
-
class ModelSearchTask extends
|
|
4404
|
+
class ModelSearchTask extends Task10 {
|
|
4797
4405
|
static type = "ModelSearchTask";
|
|
4798
4406
|
static category = "AI Model";
|
|
4799
4407
|
static title = "Model Search";
|
|
@@ -4819,10 +4427,10 @@ class ModelSearchTask extends Task13 {
|
|
|
4819
4427
|
var modelSearch = (input, config) => {
|
|
4820
4428
|
return new ModelSearchTask(config).run(input);
|
|
4821
4429
|
};
|
|
4822
|
-
|
|
4430
|
+
Workflow24.prototype.modelSearch = CreateWorkflow24(ModelSearchTask);
|
|
4823
4431
|
|
|
4824
4432
|
// src/task/ObjectDetectionTask.ts
|
|
4825
|
-
import { CreateWorkflow as
|
|
4433
|
+
import { CreateWorkflow as CreateWorkflow25, Workflow as Workflow25 } from "@workglow/task-graph";
|
|
4826
4434
|
var modelSchema19 = TypeModel("model:ObjectDetectionTask");
|
|
4827
4435
|
var detectionSchema = {
|
|
4828
4436
|
type: "object",
|
|
@@ -4902,10 +4510,10 @@ class ObjectDetectionTask extends AiVisionTask {
|
|
|
4902
4510
|
var objectDetection = (input, config) => {
|
|
4903
4511
|
return new ObjectDetectionTask(config).run(input);
|
|
4904
4512
|
};
|
|
4905
|
-
|
|
4513
|
+
Workflow25.prototype.objectDetection = CreateWorkflow25(ObjectDetectionTask);
|
|
4906
4514
|
|
|
4907
4515
|
// src/task/PoseLandmarkerTask.ts
|
|
4908
|
-
import { CreateWorkflow as
|
|
4516
|
+
import { CreateWorkflow as CreateWorkflow26, Workflow as Workflow26 } from "@workglow/task-graph";
|
|
4909
4517
|
var modelSchema20 = TypeModel("model:PoseLandmarkerTask");
|
|
4910
4518
|
var TypePoseLandmark = {
|
|
4911
4519
|
type: "object",
|
|
@@ -5064,17 +4672,15 @@ class PoseLandmarkerTask extends AiVisionTask {
|
|
|
5064
4672
|
var poseLandmarker = (input, config) => {
|
|
5065
4673
|
return new PoseLandmarkerTask(config).run(input);
|
|
5066
4674
|
};
|
|
5067
|
-
|
|
4675
|
+
Workflow26.prototype.poseLandmarker = CreateWorkflow26(PoseLandmarkerTask);
|
|
5068
4676
|
|
|
5069
4677
|
// src/task/QueryExpanderTask.ts
|
|
5070
|
-
import { CreateWorkflow as
|
|
4678
|
+
import { CreateWorkflow as CreateWorkflow27, Task as Task11, Workflow as Workflow27 } from "@workglow/task-graph";
|
|
5071
4679
|
var QueryExpansionMethod = {
|
|
5072
4680
|
MULTI_QUERY: "multi-query",
|
|
5073
|
-
|
|
5074
|
-
SYNONYMS: "synonyms",
|
|
5075
|
-
PARAPHRASE: "paraphrase"
|
|
4681
|
+
SYNONYMS: "synonyms"
|
|
5076
4682
|
};
|
|
5077
|
-
var
|
|
4683
|
+
var inputSchema9 = {
|
|
5078
4684
|
type: "object",
|
|
5079
4685
|
properties: {
|
|
5080
4686
|
query: {
|
|
@@ -5096,17 +4702,12 @@ var inputSchema12 = {
|
|
|
5096
4702
|
minimum: 1,
|
|
5097
4703
|
maximum: 10,
|
|
5098
4704
|
default: 3
|
|
5099
|
-
},
|
|
5100
|
-
model: {
|
|
5101
|
-
type: "string",
|
|
5102
|
-
title: "Model",
|
|
5103
|
-
description: "LLM model to use for expansion (for HyDE and paraphrase methods)"
|
|
5104
4705
|
}
|
|
5105
4706
|
},
|
|
5106
4707
|
required: ["query"],
|
|
5107
4708
|
additionalProperties: false
|
|
5108
4709
|
};
|
|
5109
|
-
var
|
|
4710
|
+
var outputSchema9 = {
|
|
5110
4711
|
type: "object",
|
|
5111
4712
|
properties: {
|
|
5112
4713
|
query: {
|
|
@@ -5135,31 +4736,25 @@ var outputSchema12 = {
|
|
|
5135
4736
|
additionalProperties: false
|
|
5136
4737
|
};
|
|
5137
4738
|
|
|
5138
|
-
class QueryExpanderTask extends
|
|
4739
|
+
class QueryExpanderTask extends Task11 {
|
|
5139
4740
|
static type = "QueryExpanderTask";
|
|
5140
4741
|
static category = "RAG";
|
|
5141
4742
|
static title = "Query Expander";
|
|
5142
4743
|
static description = "Expand queries to improve retrieval coverage";
|
|
5143
4744
|
static cacheable = true;
|
|
5144
4745
|
static inputSchema() {
|
|
5145
|
-
return
|
|
4746
|
+
return inputSchema9;
|
|
5146
4747
|
}
|
|
5147
4748
|
static outputSchema() {
|
|
5148
|
-
return
|
|
4749
|
+
return outputSchema9;
|
|
5149
4750
|
}
|
|
5150
4751
|
async execute(input, context) {
|
|
5151
4752
|
const { query, method = QueryExpansionMethod.MULTI_QUERY, numVariations = 3 } = input;
|
|
5152
4753
|
let queries;
|
|
5153
4754
|
switch (method) {
|
|
5154
|
-
case QueryExpansionMethod.HYDE:
|
|
5155
|
-
queries = this.hydeExpansion(query, numVariations);
|
|
5156
|
-
break;
|
|
5157
4755
|
case QueryExpansionMethod.SYNONYMS:
|
|
5158
4756
|
queries = this.synonymExpansion(query, numVariations);
|
|
5159
4757
|
break;
|
|
5160
|
-
case QueryExpansionMethod.PARAPHRASE:
|
|
5161
|
-
queries = this.paraphraseExpansion(query, numVariations);
|
|
5162
|
-
break;
|
|
5163
4758
|
case QueryExpansionMethod.MULTI_QUERY:
|
|
5164
4759
|
default:
|
|
5165
4760
|
queries = this.multiQueryExpansion(query, numVariations);
|
|
@@ -5197,174 +4792,64 @@ class QueryExpanderTask extends Task14 {
|
|
|
5197
4792
|
if (!query.toLowerCase().startsWith("explain")) {
|
|
5198
4793
|
variations.push(`Explain ${query.toLowerCase()}`);
|
|
5199
4794
|
}
|
|
5200
|
-
for (let i = 0;i < Math.min(numVariations - 1, variations.length); i++) {
|
|
5201
|
-
if (variations[i] && !queries.includes(variations[i])) {
|
|
5202
|
-
queries.push(variations[i]);
|
|
5203
|
-
}
|
|
5204
|
-
}
|
|
5205
|
-
return queries;
|
|
5206
|
-
}
|
|
5207
|
-
|
|
5208
|
-
const queries = [query];
|
|
5209
|
-
const
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
|
|
5220
|
-
|
|
5221
|
-
const
|
|
5222
|
-
|
|
5223
|
-
|
|
5224
|
-
|
|
5225
|
-
|
|
5226
|
-
|
|
5227
|
-
|
|
5228
|
-
|
|
5229
|
-
|
|
5230
|
-
|
|
5231
|
-
|
|
5232
|
-
|
|
5233
|
-
|
|
5234
|
-
|
|
5235
|
-
|
|
5236
|
-
|
|
5237
|
-
if (variationsGenerated >= numVariations - 1)
|
|
5238
|
-
break;
|
|
5239
|
-
const wordIndex = words.indexOf(word);
|
|
5240
|
-
if (wordIndex !== -1) {
|
|
5241
|
-
for (const syn of syns) {
|
|
5242
|
-
if (variationsGenerated >= numVariations - 1)
|
|
5243
|
-
break;
|
|
5244
|
-
const newWords = [...words];
|
|
5245
|
-
newWords[wordIndex] = syn;
|
|
5246
|
-
const newQuery = newWords.join(" ");
|
|
5247
|
-
const capitalizedQuery = this.preserveCapitalization(query, newQuery);
|
|
5248
|
-
if (!queries.includes(capitalizedQuery)) {
|
|
5249
|
-
queries.push(capitalizedQuery);
|
|
5250
|
-
variationsGenerated++;
|
|
5251
|
-
}
|
|
5252
|
-
}
|
|
5253
|
-
}
|
|
5254
|
-
}
|
|
5255
|
-
return queries;
|
|
5256
|
-
}
|
|
5257
|
-
paraphraseExpansion(query, numVariations) {
|
|
5258
|
-
const queries = [query];
|
|
5259
|
-
const paraphrases = [];
|
|
5260
|
-
paraphrases.push(`I need information about ${query.toLowerCase()}`);
|
|
5261
|
-
paraphrases.push(`Can you help me understand ${query.toLowerCase()}`);
|
|
5262
|
-
paraphrases.push(`I'm looking for details on ${query.toLowerCase()}`);
|
|
5263
|
-
for (let i = 0;i < Math.min(numVariations - 1, paraphrases.length); i++) {
|
|
5264
|
-
if (!queries.includes(paraphrases[i])) {
|
|
5265
|
-
queries.push(paraphrases[i]);
|
|
5266
|
-
}
|
|
5267
|
-
}
|
|
5268
|
-
return queries;
|
|
5269
|
-
}
|
|
5270
|
-
preserveCapitalization(original, modified) {
|
|
5271
|
-
if (original[0] === original[0].toUpperCase()) {
|
|
5272
|
-
return modified.charAt(0).toUpperCase() + modified.slice(1);
|
|
5273
|
-
}
|
|
5274
|
-
return modified;
|
|
5275
|
-
}
|
|
5276
|
-
}
|
|
5277
|
-
var queryExpander = (input, config) => {
|
|
5278
|
-
return new QueryExpanderTask(config).run(input);
|
|
5279
|
-
};
|
|
5280
|
-
Workflow30.prototype.queryExpander = CreateWorkflow30(QueryExpanderTask);
|
|
5281
|
-
|
|
5282
|
-
// src/task/RerankerTask.ts
|
|
5283
|
-
import { CreateWorkflow as CreateWorkflow32, Task as Task15, Workflow as Workflow32 } from "@workglow/task-graph";
|
|
5284
|
-
|
|
5285
|
-
// src/task/TextClassificationTask.ts
|
|
5286
|
-
import { CreateWorkflow as CreateWorkflow31, Workflow as Workflow31 } from "@workglow/task-graph";
|
|
5287
|
-
var modelSchema21 = TypeModel("model:TextClassificationTask");
|
|
5288
|
-
var TextClassificationInputSchema = {
|
|
5289
|
-
type: "object",
|
|
5290
|
-
properties: {
|
|
5291
|
-
text: {
|
|
5292
|
-
type: "string",
|
|
5293
|
-
title: "Text",
|
|
5294
|
-
description: "The text to classify"
|
|
5295
|
-
},
|
|
5296
|
-
candidateLabels: {
|
|
5297
|
-
type: "array",
|
|
5298
|
-
items: {
|
|
5299
|
-
type: "string"
|
|
5300
|
-
},
|
|
5301
|
-
title: "Candidate Labels",
|
|
5302
|
-
description: "List of candidate labels (optional, if provided uses zero-shot classification)",
|
|
5303
|
-
"x-ui-group": "Configuration"
|
|
5304
|
-
},
|
|
5305
|
-
maxCategories: {
|
|
5306
|
-
type: "number",
|
|
5307
|
-
minimum: 1,
|
|
5308
|
-
maximum: 1000,
|
|
5309
|
-
default: 5,
|
|
5310
|
-
title: "Max Categories",
|
|
5311
|
-
description: "The maximum number of categories to return",
|
|
5312
|
-
"x-ui-group": "Configuration"
|
|
5313
|
-
},
|
|
5314
|
-
model: modelSchema21
|
|
5315
|
-
},
|
|
5316
|
-
required: ["text", "model"],
|
|
5317
|
-
additionalProperties: false
|
|
5318
|
-
};
|
|
5319
|
-
var TextClassificationOutputSchema = {
|
|
5320
|
-
type: "object",
|
|
5321
|
-
properties: {
|
|
5322
|
-
categories: {
|
|
5323
|
-
type: "array",
|
|
5324
|
-
items: {
|
|
5325
|
-
type: "object",
|
|
5326
|
-
properties: {
|
|
5327
|
-
label: {
|
|
5328
|
-
type: "string",
|
|
5329
|
-
title: "Label",
|
|
5330
|
-
description: "The name of the category"
|
|
5331
|
-
},
|
|
5332
|
-
score: {
|
|
5333
|
-
type: "number",
|
|
5334
|
-
title: "Score",
|
|
5335
|
-
description: "The confidence score for this category"
|
|
4795
|
+
for (let i = 0;i < Math.min(numVariations - 1, variations.length); i++) {
|
|
4796
|
+
if (variations[i] && !queries.includes(variations[i])) {
|
|
4797
|
+
queries.push(variations[i]);
|
|
4798
|
+
}
|
|
4799
|
+
}
|
|
4800
|
+
return queries;
|
|
4801
|
+
}
|
|
4802
|
+
synonymExpansion(query, numVariations) {
|
|
4803
|
+
const queries = [query];
|
|
4804
|
+
const synonyms = {
|
|
4805
|
+
find: ["locate", "discover", "search for"],
|
|
4806
|
+
create: ["make", "build", "generate"],
|
|
4807
|
+
delete: ["remove", "erase", "eliminate"],
|
|
4808
|
+
update: ["modify", "change", "edit"],
|
|
4809
|
+
show: ["display", "present", "reveal"],
|
|
4810
|
+
explain: ["describe", "clarify", "elaborate"],
|
|
4811
|
+
help: ["assist", "aid", "support"],
|
|
4812
|
+
problem: ["issue", "challenge", "difficulty"],
|
|
4813
|
+
solution: ["answer", "resolution", "fix"],
|
|
4814
|
+
method: ["approach", "technique", "way"]
|
|
4815
|
+
};
|
|
4816
|
+
const words = query.toLowerCase().split(/\s+/);
|
|
4817
|
+
let variationsGenerated = 0;
|
|
4818
|
+
for (const [word, syns] of Object.entries(synonyms)) {
|
|
4819
|
+
if (variationsGenerated >= numVariations - 1)
|
|
4820
|
+
break;
|
|
4821
|
+
const wordIndex = words.indexOf(word);
|
|
4822
|
+
if (wordIndex !== -1) {
|
|
4823
|
+
for (const syn of syns) {
|
|
4824
|
+
if (variationsGenerated >= numVariations - 1)
|
|
4825
|
+
break;
|
|
4826
|
+
const newWords = [...words];
|
|
4827
|
+
newWords[wordIndex] = syn;
|
|
4828
|
+
const capitalizedQuery = this.preserveCapitalization(query, newWords.join(" "));
|
|
4829
|
+
if (!queries.includes(capitalizedQuery)) {
|
|
4830
|
+
queries.push(capitalizedQuery);
|
|
4831
|
+
variationsGenerated++;
|
|
5336
4832
|
}
|
|
5337
|
-
}
|
|
5338
|
-
|
|
5339
|
-
additionalProperties: false
|
|
5340
|
-
},
|
|
5341
|
-
title: "Categories",
|
|
5342
|
-
description: "The classification categories with their scores"
|
|
4833
|
+
}
|
|
4834
|
+
}
|
|
5343
4835
|
}
|
|
5344
|
-
|
|
5345
|
-
required: ["categories"],
|
|
5346
|
-
additionalProperties: false
|
|
5347
|
-
};
|
|
5348
|
-
|
|
5349
|
-
class TextClassificationTask extends AiTask {
|
|
5350
|
-
static type = "TextClassificationTask";
|
|
5351
|
-
static category = "AI Text Model";
|
|
5352
|
-
static title = "Text Classifier";
|
|
5353
|
-
static description = "Classifies text into categories using language models. Supports zero-shot classification when candidate labels are provided.";
|
|
5354
|
-
static inputSchema() {
|
|
5355
|
-
return TextClassificationInputSchema;
|
|
4836
|
+
return queries;
|
|
5356
4837
|
}
|
|
5357
|
-
|
|
5358
|
-
|
|
4838
|
+
preserveCapitalization(original, modified) {
|
|
4839
|
+
if (original[0] === original[0].toUpperCase()) {
|
|
4840
|
+
return modified.charAt(0).toUpperCase() + modified.slice(1);
|
|
4841
|
+
}
|
|
4842
|
+
return modified;
|
|
5359
4843
|
}
|
|
5360
4844
|
}
|
|
5361
|
-
var
|
|
5362
|
-
return new
|
|
4845
|
+
var queryExpander = (input, config) => {
|
|
4846
|
+
return new QueryExpanderTask(config).run(input);
|
|
5363
4847
|
};
|
|
5364
|
-
|
|
4848
|
+
Workflow27.prototype.queryExpander = CreateWorkflow27(QueryExpanderTask);
|
|
5365
4849
|
|
|
5366
4850
|
// src/task/RerankerTask.ts
|
|
5367
|
-
|
|
4851
|
+
import { CreateWorkflow as CreateWorkflow28, Task as Task12, Workflow as Workflow28 } from "@workglow/task-graph";
|
|
4852
|
+
var inputSchema10 = {
|
|
5368
4853
|
type: "object",
|
|
5369
4854
|
properties: {
|
|
5370
4855
|
query: {
|
|
@@ -5402,20 +4887,16 @@ var inputSchema13 = {
|
|
|
5402
4887
|
},
|
|
5403
4888
|
method: {
|
|
5404
4889
|
type: "string",
|
|
5405
|
-
enum: ["
|
|
4890
|
+
enum: ["reciprocal-rank-fusion", "simple"],
|
|
5406
4891
|
title: "Reranking Method",
|
|
5407
4892
|
description: "Method to use for reranking",
|
|
5408
4893
|
default: "simple"
|
|
5409
|
-
}
|
|
5410
|
-
model: TypeModel("model:RerankerTask", {
|
|
5411
|
-
title: "Reranker Model",
|
|
5412
|
-
description: "Cross-encoder model to use for reranking"
|
|
5413
|
-
})
|
|
4894
|
+
}
|
|
5414
4895
|
},
|
|
5415
4896
|
required: ["query", "chunks"],
|
|
5416
4897
|
additionalProperties: false
|
|
5417
4898
|
};
|
|
5418
|
-
var
|
|
4899
|
+
var outputSchema10 = {
|
|
5419
4900
|
type: "object",
|
|
5420
4901
|
properties: {
|
|
5421
4902
|
chunks: {
|
|
@@ -5456,28 +4937,22 @@ var outputSchema13 = {
|
|
|
5456
4937
|
additionalProperties: false
|
|
5457
4938
|
};
|
|
5458
4939
|
|
|
5459
|
-
class RerankerTask extends
|
|
4940
|
+
class RerankerTask extends Task12 {
|
|
5460
4941
|
static type = "RerankerTask";
|
|
5461
4942
|
static category = "RAG";
|
|
5462
4943
|
static title = "Reranker";
|
|
5463
4944
|
static description = "Rerank retrieved chunks to improve relevance";
|
|
5464
4945
|
static cacheable = true;
|
|
5465
4946
|
static inputSchema() {
|
|
5466
|
-
return
|
|
4947
|
+
return inputSchema10;
|
|
5467
4948
|
}
|
|
5468
4949
|
static outputSchema() {
|
|
5469
|
-
return
|
|
4950
|
+
return outputSchema10;
|
|
5470
4951
|
}
|
|
5471
4952
|
async execute(input, context) {
|
|
5472
|
-
const { query, chunks, scores = [], metadata = [], topK, method = "simple"
|
|
4953
|
+
const { query, chunks, scores = [], metadata = [], topK, method = "simple" } = input;
|
|
5473
4954
|
let rankedItems;
|
|
5474
4955
|
switch (method) {
|
|
5475
|
-
case "cross-encoder":
|
|
5476
|
-
if (!model) {
|
|
5477
|
-
throw new Error("No cross-encoder model found. Please provide a model with format 'model:RerankerTask'.");
|
|
5478
|
-
}
|
|
5479
|
-
rankedItems = await this.crossEncoderRerank(query, chunks, scores, metadata, model, context);
|
|
5480
|
-
break;
|
|
5481
4956
|
case "reciprocal-rank-fusion":
|
|
5482
4957
|
rankedItems = this.reciprocalRankFusion(chunks, scores, metadata);
|
|
5483
4958
|
break;
|
|
@@ -5497,44 +4972,6 @@ class RerankerTask extends Task15 {
|
|
|
5497
4972
|
count: rankedItems.length
|
|
5498
4973
|
};
|
|
5499
4974
|
}
|
|
5500
|
-
async crossEncoderRerank(query, chunks, scores, metadata, model, context) {
|
|
5501
|
-
if (chunks.length === 0) {
|
|
5502
|
-
return [];
|
|
5503
|
-
}
|
|
5504
|
-
if (!model) {
|
|
5505
|
-
throw new Error("No cross-encoder model found. Please provide a model or register a TextClassificationTask model.");
|
|
5506
|
-
}
|
|
5507
|
-
const items = await Promise.all(chunks.map(async (chunk, index) => {
|
|
5508
|
-
const pairText = `${query} [SEP] ${chunk}`;
|
|
5509
|
-
const task = context.own(new TextClassificationTask({ defaults: { model } }));
|
|
5510
|
-
const result = await task.run({ text: pairText, maxCategories: 2 });
|
|
5511
|
-
const crossScore = this.extractCrossEncoderScore(result.categories);
|
|
5512
|
-
return {
|
|
5513
|
-
chunk,
|
|
5514
|
-
score: Number.isFinite(crossScore) ? crossScore : scores[index] || 0,
|
|
5515
|
-
metadata: metadata[index],
|
|
5516
|
-
originalIndex: index
|
|
5517
|
-
};
|
|
5518
|
-
}));
|
|
5519
|
-
items.sort((a, b) => b.score - a.score);
|
|
5520
|
-
return items;
|
|
5521
|
-
}
|
|
5522
|
-
extractCrossEncoderScore(categories) {
|
|
5523
|
-
if (!categories || categories.length === 0) {
|
|
5524
|
-
return 0;
|
|
5525
|
-
}
|
|
5526
|
-
const preferred = categories.find((category) => /^(label_1|positive|relevant|yes|true)$/i.test(category.label));
|
|
5527
|
-
if (preferred) {
|
|
5528
|
-
return preferred.score;
|
|
5529
|
-
}
|
|
5530
|
-
let best = categories[0].score;
|
|
5531
|
-
for (let i = 1;i < categories.length; i++) {
|
|
5532
|
-
if (categories[i].score > best) {
|
|
5533
|
-
best = categories[i].score;
|
|
5534
|
-
}
|
|
5535
|
-
}
|
|
5536
|
-
return best;
|
|
5537
|
-
}
|
|
5538
4975
|
simpleRerank(query, chunks, scores, metadata) {
|
|
5539
4976
|
const queryLower = query.toLowerCase();
|
|
5540
4977
|
const queryWords = queryLower.split(/\s+/).filter((w) => w.length > 0);
|
|
@@ -5542,7 +4979,6 @@ class RerankerTask extends Task15 {
|
|
|
5542
4979
|
const chunkLower = chunk.toLowerCase();
|
|
5543
4980
|
const initialScore = scores[index] || 0;
|
|
5544
4981
|
let keywordScore = 0;
|
|
5545
|
-
let exactMatchBonus = 0;
|
|
5546
4982
|
for (const word of queryWords) {
|
|
5547
4983
|
const regex = new RegExp(word, "gi");
|
|
5548
4984
|
const matches = chunkLower.match(regex);
|
|
@@ -5550,33 +4986,23 @@ class RerankerTask extends Task15 {
|
|
|
5550
4986
|
keywordScore += matches.length;
|
|
5551
4987
|
}
|
|
5552
4988
|
}
|
|
5553
|
-
|
|
5554
|
-
exactMatchBonus = 0.5;
|
|
5555
|
-
}
|
|
4989
|
+
const exactMatchBonus = chunkLower.includes(queryLower) ? 0.5 : 0;
|
|
5556
4990
|
const normalizedKeywordScore = Math.min(keywordScore / (queryWords.length * 3), 1);
|
|
5557
4991
|
const positionPenalty = Math.log(index + 1) / 10;
|
|
5558
4992
|
const combinedScore = initialScore * 0.4 + normalizedKeywordScore * 0.4 + exactMatchBonus * 0.2 - positionPenalty;
|
|
5559
|
-
return {
|
|
5560
|
-
chunk,
|
|
5561
|
-
score: combinedScore,
|
|
5562
|
-
metadata: metadata[index],
|
|
5563
|
-
originalIndex: index
|
|
5564
|
-
};
|
|
4993
|
+
return { chunk, score: combinedScore, metadata: metadata[index], originalIndex: index };
|
|
5565
4994
|
});
|
|
5566
4995
|
items.sort((a, b) => b.score - a.score);
|
|
5567
4996
|
return items;
|
|
5568
4997
|
}
|
|
5569
4998
|
reciprocalRankFusion(chunks, scores, metadata) {
|
|
5570
4999
|
const k = 60;
|
|
5571
|
-
const items = chunks.map((chunk, index) => {
|
|
5572
|
-
|
|
5573
|
-
|
|
5574
|
-
|
|
5575
|
-
|
|
5576
|
-
|
|
5577
|
-
originalIndex: index
|
|
5578
|
-
};
|
|
5579
|
-
});
|
|
5000
|
+
const items = chunks.map((chunk, index) => ({
|
|
5001
|
+
chunk,
|
|
5002
|
+
score: 1 / (k + index + 1),
|
|
5003
|
+
metadata: metadata[index],
|
|
5004
|
+
originalIndex: index
|
|
5005
|
+
}));
|
|
5580
5006
|
items.sort((a, b) => b.score - a.score);
|
|
5581
5007
|
return items;
|
|
5582
5008
|
}
|
|
@@ -5584,13 +5010,13 @@ class RerankerTask extends Task15 {
|
|
|
5584
5010
|
var reranker = (input, config) => {
|
|
5585
5011
|
return new RerankerTask(config).run(input);
|
|
5586
5012
|
};
|
|
5587
|
-
|
|
5013
|
+
Workflow28.prototype.reranker = CreateWorkflow28(RerankerTask);
|
|
5588
5014
|
|
|
5589
5015
|
// src/task/StructuralParserTask.ts
|
|
5590
5016
|
import { StructuralParser } from "@workglow/knowledge-base";
|
|
5591
|
-
import { CreateWorkflow as
|
|
5017
|
+
import { CreateWorkflow as CreateWorkflow29, Task as Task13, Workflow as Workflow29 } from "@workglow/task-graph";
|
|
5592
5018
|
import { uuid4 as uuid42 } from "@workglow/util";
|
|
5593
|
-
var
|
|
5019
|
+
var inputSchema11 = {
|
|
5594
5020
|
type: "object",
|
|
5595
5021
|
properties: {
|
|
5596
5022
|
text: {
|
|
@@ -5624,7 +5050,7 @@ var inputSchema14 = {
|
|
|
5624
5050
|
required: ["text", "title"],
|
|
5625
5051
|
additionalProperties: false
|
|
5626
5052
|
};
|
|
5627
|
-
var
|
|
5053
|
+
var outputSchema11 = {
|
|
5628
5054
|
type: "object",
|
|
5629
5055
|
properties: {
|
|
5630
5056
|
doc_id: {
|
|
@@ -5648,17 +5074,17 @@ var outputSchema14 = {
|
|
|
5648
5074
|
additionalProperties: false
|
|
5649
5075
|
};
|
|
5650
5076
|
|
|
5651
|
-
class StructuralParserTask extends
|
|
5077
|
+
class StructuralParserTask extends Task13 {
|
|
5652
5078
|
static type = "StructuralParserTask";
|
|
5653
5079
|
static category = "Document";
|
|
5654
5080
|
static title = "Structural Parser";
|
|
5655
5081
|
static description = "Parse documents into hierarchical tree structure";
|
|
5656
5082
|
static cacheable = true;
|
|
5657
5083
|
static inputSchema() {
|
|
5658
|
-
return
|
|
5084
|
+
return inputSchema11;
|
|
5659
5085
|
}
|
|
5660
5086
|
static outputSchema() {
|
|
5661
|
-
return
|
|
5087
|
+
return outputSchema11;
|
|
5662
5088
|
}
|
|
5663
5089
|
async execute(input, context) {
|
|
5664
5090
|
const { text, title, format = "auto", sourceUri, doc_id: providedDocId } = input;
|
|
@@ -5691,16 +5117,16 @@ class StructuralParserTask extends Task16 {
|
|
|
5691
5117
|
var structuralParser = (input, config) => {
|
|
5692
5118
|
return new StructuralParserTask(config).run(input);
|
|
5693
5119
|
};
|
|
5694
|
-
|
|
5120
|
+
Workflow29.prototype.structuralParser = CreateWorkflow29(StructuralParserTask);
|
|
5695
5121
|
|
|
5696
5122
|
// src/task/StructuredGenerationTask.ts
|
|
5697
|
-
import { CreateWorkflow as
|
|
5123
|
+
import { CreateWorkflow as CreateWorkflow30, TaskConfigurationError as TaskConfigurationError4, TaskError, Workflow as Workflow30 } from "@workglow/task-graph";
|
|
5698
5124
|
import { compileSchema as compileSchema2 } from "@workglow/util/schema";
|
|
5699
|
-
var
|
|
5125
|
+
var modelSchema21 = TypeModel("model:StructuredGenerationTask");
|
|
5700
5126
|
var StructuredGenerationInputSchema = {
|
|
5701
5127
|
type: "object",
|
|
5702
5128
|
properties: {
|
|
5703
|
-
model:
|
|
5129
|
+
model: modelSchema21,
|
|
5704
5130
|
prompt: {
|
|
5705
5131
|
type: "string",
|
|
5706
5132
|
title: "Prompt",
|
|
@@ -5868,17 +5294,18 @@ class StructuredGenerationTask extends StreamingAiTask {
|
|
|
5868
5294
|
var structuredGeneration = (input, config) => {
|
|
5869
5295
|
return new StructuredGenerationTask(config).run(input);
|
|
5870
5296
|
};
|
|
5871
|
-
|
|
5297
|
+
Workflow30.prototype.structuredGeneration = CreateWorkflow30(StructuredGenerationTask);
|
|
5872
5298
|
|
|
5873
5299
|
// src/task/TextChunkerTask.ts
|
|
5874
|
-
import {
|
|
5300
|
+
import { ChunkRecordArraySchema as ChunkRecordArraySchema3 } from "@workglow/knowledge-base";
|
|
5301
|
+
import { CreateWorkflow as CreateWorkflow31, Task as Task14, Workflow as Workflow31 } from "@workglow/task-graph";
|
|
5875
5302
|
var ChunkingStrategy = {
|
|
5876
5303
|
FIXED: "fixed",
|
|
5877
5304
|
SENTENCE: "sentence",
|
|
5878
5305
|
PARAGRAPH: "paragraph",
|
|
5879
5306
|
SEMANTIC: "semantic"
|
|
5880
5307
|
};
|
|
5881
|
-
var
|
|
5308
|
+
var inputSchema12 = {
|
|
5882
5309
|
type: "object",
|
|
5883
5310
|
properties: {
|
|
5884
5311
|
text: {
|
|
@@ -5886,6 +5313,11 @@ var inputSchema15 = {
|
|
|
5886
5313
|
title: "Text",
|
|
5887
5314
|
description: "The text to chunk"
|
|
5888
5315
|
},
|
|
5316
|
+
doc_id: {
|
|
5317
|
+
type: "string",
|
|
5318
|
+
title: "Document ID",
|
|
5319
|
+
description: "Optional document ID stamped onto each chunk. When omitted, chunks are emitted without a doc_id and the output also has no doc_id."
|
|
5320
|
+
},
|
|
5889
5321
|
chunkSize: {
|
|
5890
5322
|
type: "number",
|
|
5891
5323
|
title: "Chunk Size",
|
|
@@ -5911,88 +5343,99 @@ var inputSchema15 = {
|
|
|
5911
5343
|
required: ["text"],
|
|
5912
5344
|
additionalProperties: false
|
|
5913
5345
|
};
|
|
5914
|
-
var
|
|
5346
|
+
var outputSchema12 = {
|
|
5915
5347
|
type: "object",
|
|
5916
5348
|
properties: {
|
|
5917
|
-
|
|
5349
|
+
doc_id: {
|
|
5350
|
+
type: "string",
|
|
5351
|
+
title: "Document ID",
|
|
5352
|
+
description: "The document ID (only emitted when provided in input)"
|
|
5353
|
+
},
|
|
5354
|
+
chunks: ChunkRecordArraySchema3,
|
|
5355
|
+
text: {
|
|
5918
5356
|
type: "array",
|
|
5919
5357
|
items: { type: "string" },
|
|
5920
|
-
title: "
|
|
5921
|
-
description: "
|
|
5358
|
+
title: "Texts",
|
|
5359
|
+
description: "Chunk texts (for TextEmbeddingTask)"
|
|
5922
5360
|
},
|
|
5923
|
-
|
|
5924
|
-
type: "
|
|
5925
|
-
|
|
5926
|
-
|
|
5927
|
-
properties: {
|
|
5928
|
-
index: { type: "number" },
|
|
5929
|
-
startChar: { type: "number" },
|
|
5930
|
-
endChar: { type: "number" },
|
|
5931
|
-
length: { type: "number" }
|
|
5932
|
-
},
|
|
5933
|
-
additionalProperties: false
|
|
5934
|
-
},
|
|
5935
|
-
title: "Chunk Metadata",
|
|
5936
|
-
description: "Metadata for each chunk"
|
|
5361
|
+
count: {
|
|
5362
|
+
type: "number",
|
|
5363
|
+
title: "Count",
|
|
5364
|
+
description: "Number of chunks generated"
|
|
5937
5365
|
}
|
|
5938
5366
|
},
|
|
5939
|
-
required: ["chunks", "
|
|
5367
|
+
required: ["chunks", "text", "count"],
|
|
5940
5368
|
additionalProperties: false
|
|
5941
5369
|
};
|
|
5942
5370
|
|
|
5943
|
-
class TextChunkerTask extends
|
|
5371
|
+
class TextChunkerTask extends Task14 {
|
|
5944
5372
|
static type = "TextChunkerTask";
|
|
5945
5373
|
static category = "Document";
|
|
5946
5374
|
static title = "Text Chunker";
|
|
5947
5375
|
static description = "Splits text into chunks using various strategies (fixed, sentence, paragraph)";
|
|
5948
5376
|
static cacheable = true;
|
|
5949
5377
|
static inputSchema() {
|
|
5950
|
-
return
|
|
5378
|
+
return inputSchema12;
|
|
5951
5379
|
}
|
|
5952
5380
|
static outputSchema() {
|
|
5953
|
-
return
|
|
5381
|
+
return outputSchema12;
|
|
5954
5382
|
}
|
|
5955
5383
|
async execute(input, context) {
|
|
5956
|
-
const {
|
|
5957
|
-
|
|
5958
|
-
|
|
5384
|
+
const {
|
|
5385
|
+
text,
|
|
5386
|
+
doc_id,
|
|
5387
|
+
chunkSize = 512,
|
|
5388
|
+
chunkOverlap = 50,
|
|
5389
|
+
strategy = ChunkingStrategy.FIXED
|
|
5390
|
+
} = input;
|
|
5391
|
+
let rawChunks;
|
|
5959
5392
|
switch (strategy) {
|
|
5960
5393
|
case ChunkingStrategy.SENTENCE:
|
|
5961
|
-
|
|
5394
|
+
case ChunkingStrategy.SEMANTIC:
|
|
5395
|
+
rawChunks = this.chunkBySentence(text, chunkSize, chunkOverlap);
|
|
5962
5396
|
break;
|
|
5963
5397
|
case ChunkingStrategy.PARAGRAPH:
|
|
5964
|
-
|
|
5965
|
-
break;
|
|
5966
|
-
case ChunkingStrategy.SEMANTIC:
|
|
5967
|
-
({ chunks, metadata } = this.chunkBySentence(text, chunkSize, chunkOverlap));
|
|
5398
|
+
rawChunks = this.chunkByParagraph(text, chunkSize, chunkOverlap);
|
|
5968
5399
|
break;
|
|
5969
5400
|
case ChunkingStrategy.FIXED:
|
|
5970
5401
|
default:
|
|
5971
|
-
|
|
5402
|
+
rawChunks = this.chunkFixed(text, chunkSize, chunkOverlap);
|
|
5972
5403
|
break;
|
|
5973
5404
|
}
|
|
5974
|
-
|
|
5405
|
+
const nodePath = doc_id ? [doc_id] : [];
|
|
5406
|
+
const chunks = rawChunks.map((raw, index) => ({
|
|
5407
|
+
chunkId: doc_id ? `${doc_id}:${index}` : `chunk:${index}:${raw.startChar}`,
|
|
5408
|
+
doc_id: doc_id ?? "",
|
|
5409
|
+
text: raw.text,
|
|
5410
|
+
nodePath,
|
|
5411
|
+
depth: nodePath.length,
|
|
5412
|
+
...doc_id ? { leafNodeId: doc_id } : {},
|
|
5413
|
+
index,
|
|
5414
|
+
startChar: raw.startChar,
|
|
5415
|
+
endChar: raw.endChar
|
|
5416
|
+
}));
|
|
5417
|
+
const output = {
|
|
5418
|
+
chunks,
|
|
5419
|
+
text: chunks.map((c) => c.text),
|
|
5420
|
+
count: chunks.length
|
|
5421
|
+
};
|
|
5422
|
+
if (doc_id)
|
|
5423
|
+
output.doc_id = doc_id;
|
|
5424
|
+
return output;
|
|
5975
5425
|
}
|
|
5976
5426
|
chunkFixed(text, chunkSize, chunkOverlap) {
|
|
5977
5427
|
const chunks = [];
|
|
5978
|
-
const metadata = [];
|
|
5979
5428
|
let startChar = 0;
|
|
5980
|
-
let index = 0;
|
|
5981
5429
|
while (startChar < text.length) {
|
|
5982
5430
|
const endChar = Math.min(startChar + chunkSize, text.length);
|
|
5983
|
-
|
|
5984
|
-
|
|
5985
|
-
metadata.push({
|
|
5986
|
-
index,
|
|
5431
|
+
chunks.push({
|
|
5432
|
+
text: text.substring(startChar, endChar),
|
|
5987
5433
|
startChar,
|
|
5988
|
-
endChar
|
|
5989
|
-
length: chunk.length
|
|
5434
|
+
endChar
|
|
5990
5435
|
});
|
|
5991
|
-
|
|
5992
|
-
startChar += step;
|
|
5993
|
-
index++;
|
|
5436
|
+
startChar += Math.max(1, chunkSize - chunkOverlap);
|
|
5994
5437
|
}
|
|
5995
|
-
return
|
|
5438
|
+
return chunks;
|
|
5996
5439
|
}
|
|
5997
5440
|
chunkBySentence(text, chunkSize, chunkOverlap) {
|
|
5998
5441
|
const sentenceRegex = /[.!?]+[\s\n]+/g;
|
|
@@ -6001,8 +5444,7 @@ class TextChunkerTask extends Task17 {
|
|
|
6001
5444
|
let lastIndex = 0;
|
|
6002
5445
|
let match;
|
|
6003
5446
|
while ((match = sentenceRegex.exec(text)) !== null) {
|
|
6004
|
-
|
|
6005
|
-
sentences.push(sentence);
|
|
5447
|
+
sentences.push(text.substring(lastIndex, match.index + match[0].length));
|
|
6006
5448
|
sentenceStarts.push(lastIndex);
|
|
6007
5449
|
lastIndex = match.index + match[0].length;
|
|
6008
5450
|
}
|
|
@@ -6011,22 +5453,17 @@ class TextChunkerTask extends Task17 {
|
|
|
6011
5453
|
sentenceStarts.push(lastIndex);
|
|
6012
5454
|
}
|
|
6013
5455
|
const chunks = [];
|
|
6014
|
-
const metadata = [];
|
|
6015
5456
|
let currentChunk = "";
|
|
6016
5457
|
let currentStartChar = 0;
|
|
6017
|
-
let index = 0;
|
|
6018
5458
|
for (let i = 0;i < sentences.length; i++) {
|
|
6019
5459
|
const sentence = sentences[i];
|
|
6020
5460
|
const sentenceStart = sentenceStarts[i];
|
|
6021
5461
|
if (currentChunk.length > 0 && currentChunk.length + sentence.length > chunkSize) {
|
|
6022
|
-
chunks.push(
|
|
6023
|
-
|
|
6024
|
-
index,
|
|
5462
|
+
chunks.push({
|
|
5463
|
+
text: currentChunk.trim(),
|
|
6025
5464
|
startChar: currentStartChar,
|
|
6026
|
-
endChar: currentStartChar + currentChunk.length
|
|
6027
|
-
length: currentChunk.trim().length
|
|
5465
|
+
endChar: currentStartChar + currentChunk.length
|
|
6028
5466
|
});
|
|
6029
|
-
index++;
|
|
6030
5467
|
if (chunkOverlap > 0) {
|
|
6031
5468
|
let overlapText = "";
|
|
6032
5469
|
let j = i - 1;
|
|
@@ -6048,37 +5485,30 @@ class TextChunkerTask extends Task17 {
|
|
|
6048
5485
|
}
|
|
6049
5486
|
}
|
|
6050
5487
|
if (currentChunk.length > 0) {
|
|
6051
|
-
chunks.push(
|
|
6052
|
-
|
|
6053
|
-
index,
|
|
5488
|
+
chunks.push({
|
|
5489
|
+
text: currentChunk.trim(),
|
|
6054
5490
|
startChar: currentStartChar,
|
|
6055
|
-
endChar: currentStartChar + currentChunk.length
|
|
6056
|
-
length: currentChunk.trim().length
|
|
5491
|
+
endChar: currentStartChar + currentChunk.length
|
|
6057
5492
|
});
|
|
6058
5493
|
}
|
|
6059
|
-
return
|
|
5494
|
+
return chunks;
|
|
6060
5495
|
}
|
|
6061
5496
|
chunkByParagraph(text, chunkSize, chunkOverlap) {
|
|
6062
5497
|
const paragraphs = text.split(/\n\s*\n/).filter((p) => p.trim().length > 0);
|
|
6063
5498
|
const chunks = [];
|
|
6064
|
-
const metadata = [];
|
|
6065
5499
|
let currentChunk = "";
|
|
6066
5500
|
let currentStartChar = 0;
|
|
6067
|
-
let index = 0;
|
|
6068
5501
|
let charPosition = 0;
|
|
6069
5502
|
for (let i = 0;i < paragraphs.length; i++) {
|
|
6070
5503
|
const paragraph = paragraphs[i].trim();
|
|
6071
5504
|
const paragraphStart = text.indexOf(paragraph, charPosition);
|
|
6072
5505
|
charPosition = paragraphStart + paragraph.length;
|
|
6073
5506
|
if (currentChunk.length > 0 && currentChunk.length + paragraph.length + 2 > chunkSize) {
|
|
6074
|
-
chunks.push(
|
|
6075
|
-
|
|
6076
|
-
index,
|
|
5507
|
+
chunks.push({
|
|
5508
|
+
text: currentChunk.trim(),
|
|
6077
5509
|
startChar: currentStartChar,
|
|
6078
|
-
endChar: currentStartChar + currentChunk.length
|
|
6079
|
-
length: currentChunk.trim().length
|
|
5510
|
+
endChar: currentStartChar + currentChunk.length
|
|
6080
5511
|
});
|
|
6081
|
-
index++;
|
|
6082
5512
|
if (chunkOverlap > 0 && i > 0) {
|
|
6083
5513
|
let overlapText = "";
|
|
6084
5514
|
let j = i - 1;
|
|
@@ -6106,24 +5536,103 @@ class TextChunkerTask extends Task17 {
|
|
|
6106
5536
|
}
|
|
6107
5537
|
}
|
|
6108
5538
|
if (currentChunk.length > 0) {
|
|
6109
|
-
chunks.push(
|
|
6110
|
-
|
|
6111
|
-
index,
|
|
5539
|
+
chunks.push({
|
|
5540
|
+
text: currentChunk.trim(),
|
|
6112
5541
|
startChar: currentStartChar,
|
|
6113
|
-
endChar: currentStartChar + currentChunk.length
|
|
6114
|
-
length: currentChunk.trim().length
|
|
5542
|
+
endChar: currentStartChar + currentChunk.length
|
|
6115
5543
|
});
|
|
6116
5544
|
}
|
|
6117
|
-
return
|
|
5545
|
+
return chunks;
|
|
6118
5546
|
}
|
|
6119
5547
|
}
|
|
6120
5548
|
var textChunker = (input, config) => {
|
|
6121
5549
|
return new TextChunkerTask(config).run(input);
|
|
6122
5550
|
};
|
|
6123
|
-
|
|
5551
|
+
Workflow31.prototype.textChunker = CreateWorkflow31(TextChunkerTask);
|
|
5552
|
+
|
|
5553
|
+
// src/task/TextClassificationTask.ts
|
|
5554
|
+
import { CreateWorkflow as CreateWorkflow32, Workflow as Workflow32 } from "@workglow/task-graph";
|
|
5555
|
+
var modelSchema22 = TypeModel("model:TextClassificationTask");
|
|
5556
|
+
var TextClassificationInputSchema = {
|
|
5557
|
+
type: "object",
|
|
5558
|
+
properties: {
|
|
5559
|
+
text: {
|
|
5560
|
+
type: "string",
|
|
5561
|
+
title: "Text",
|
|
5562
|
+
description: "The text to classify"
|
|
5563
|
+
},
|
|
5564
|
+
candidateLabels: {
|
|
5565
|
+
type: "array",
|
|
5566
|
+
items: {
|
|
5567
|
+
type: "string"
|
|
5568
|
+
},
|
|
5569
|
+
title: "Candidate Labels",
|
|
5570
|
+
description: "List of candidate labels (optional, if provided uses zero-shot classification)",
|
|
5571
|
+
"x-ui-group": "Configuration"
|
|
5572
|
+
},
|
|
5573
|
+
maxCategories: {
|
|
5574
|
+
type: "number",
|
|
5575
|
+
minimum: 1,
|
|
5576
|
+
maximum: 1000,
|
|
5577
|
+
default: 5,
|
|
5578
|
+
title: "Max Categories",
|
|
5579
|
+
description: "The maximum number of categories to return",
|
|
5580
|
+
"x-ui-group": "Configuration"
|
|
5581
|
+
},
|
|
5582
|
+
model: modelSchema22
|
|
5583
|
+
},
|
|
5584
|
+
required: ["text", "model"],
|
|
5585
|
+
additionalProperties: false
|
|
5586
|
+
};
|
|
5587
|
+
var TextClassificationOutputSchema = {
|
|
5588
|
+
type: "object",
|
|
5589
|
+
properties: {
|
|
5590
|
+
categories: {
|
|
5591
|
+
type: "array",
|
|
5592
|
+
items: {
|
|
5593
|
+
type: "object",
|
|
5594
|
+
properties: {
|
|
5595
|
+
label: {
|
|
5596
|
+
type: "string",
|
|
5597
|
+
title: "Label",
|
|
5598
|
+
description: "The name of the category"
|
|
5599
|
+
},
|
|
5600
|
+
score: {
|
|
5601
|
+
type: "number",
|
|
5602
|
+
title: "Score",
|
|
5603
|
+
description: "The confidence score for this category"
|
|
5604
|
+
}
|
|
5605
|
+
},
|
|
5606
|
+
required: ["label", "score"],
|
|
5607
|
+
additionalProperties: false
|
|
5608
|
+
},
|
|
5609
|
+
title: "Categories",
|
|
5610
|
+
description: "The classification categories with their scores"
|
|
5611
|
+
}
|
|
5612
|
+
},
|
|
5613
|
+
required: ["categories"],
|
|
5614
|
+
additionalProperties: false
|
|
5615
|
+
};
|
|
5616
|
+
|
|
5617
|
+
class TextClassificationTask extends AiTask {
|
|
5618
|
+
static type = "TextClassificationTask";
|
|
5619
|
+
static category = "AI Text Model";
|
|
5620
|
+
static title = "Text Classifier";
|
|
5621
|
+
static description = "Classifies text into categories using language models. Supports zero-shot classification when candidate labels are provided.";
|
|
5622
|
+
static inputSchema() {
|
|
5623
|
+
return TextClassificationInputSchema;
|
|
5624
|
+
}
|
|
5625
|
+
static outputSchema() {
|
|
5626
|
+
return TextClassificationOutputSchema;
|
|
5627
|
+
}
|
|
5628
|
+
}
|
|
5629
|
+
var textClassification = (input, config) => {
|
|
5630
|
+
return new TextClassificationTask(config).run(input);
|
|
5631
|
+
};
|
|
5632
|
+
Workflow32.prototype.textClassification = CreateWorkflow32(TextClassificationTask);
|
|
6124
5633
|
|
|
6125
5634
|
// src/task/TextFillMaskTask.ts
|
|
6126
|
-
import { CreateWorkflow as
|
|
5635
|
+
import { CreateWorkflow as CreateWorkflow33, Workflow as Workflow33 } from "@workglow/task-graph";
|
|
6127
5636
|
var modelSchema23 = TypeModel("model:TextFillMaskTask");
|
|
6128
5637
|
var TextFillMaskInputSchema = {
|
|
6129
5638
|
type: "object",
|
|
@@ -6188,10 +5697,10 @@ class TextFillMaskTask extends AiTask {
|
|
|
6188
5697
|
var textFillMask = (input, config) => {
|
|
6189
5698
|
return new TextFillMaskTask(config).run(input);
|
|
6190
5699
|
};
|
|
6191
|
-
|
|
5700
|
+
Workflow33.prototype.textFillMask = CreateWorkflow33(TextFillMaskTask);
|
|
6192
5701
|
|
|
6193
5702
|
// src/task/TextGenerationTask.ts
|
|
6194
|
-
import { CreateWorkflow as
|
|
5703
|
+
import { CreateWorkflow as CreateWorkflow34, Workflow as Workflow34 } from "@workglow/task-graph";
|
|
6195
5704
|
var generatedTextSchema2 = {
|
|
6196
5705
|
type: "string",
|
|
6197
5706
|
title: "Text",
|
|
@@ -6276,10 +5785,10 @@ class TextGenerationTask extends StreamingAiTask {
|
|
|
6276
5785
|
var textGeneration = (input, config) => {
|
|
6277
5786
|
return new TextGenerationTask(config).run(input);
|
|
6278
5787
|
};
|
|
6279
|
-
|
|
5788
|
+
Workflow34.prototype.textGeneration = CreateWorkflow34(TextGenerationTask);
|
|
6280
5789
|
|
|
6281
5790
|
// src/task/TextLanguageDetectionTask.ts
|
|
6282
|
-
import { CreateWorkflow as
|
|
5791
|
+
import { CreateWorkflow as CreateWorkflow35, Workflow as Workflow35 } from "@workglow/task-graph";
|
|
6283
5792
|
var modelSchema25 = TypeModel("model:TextLanguageDetectionTask");
|
|
6284
5793
|
var TextLanguageDetectionInputSchema = {
|
|
6285
5794
|
type: "object",
|
|
@@ -6347,10 +5856,10 @@ class TextLanguageDetectionTask extends AiTask {
|
|
|
6347
5856
|
var textLanguageDetection = (input, config) => {
|
|
6348
5857
|
return new TextLanguageDetectionTask(config).run(input);
|
|
6349
5858
|
};
|
|
6350
|
-
|
|
5859
|
+
Workflow35.prototype.textLanguageDetection = CreateWorkflow35(TextLanguageDetectionTask);
|
|
6351
5860
|
|
|
6352
5861
|
// src/task/TextQuestionAnswerTask.ts
|
|
6353
|
-
import { CreateWorkflow as
|
|
5862
|
+
import { CreateWorkflow as CreateWorkflow36, Workflow as Workflow36 } from "@workglow/task-graph";
|
|
6354
5863
|
var contextSchema = {
|
|
6355
5864
|
type: "string",
|
|
6356
5865
|
title: "Context",
|
|
@@ -6402,10 +5911,10 @@ class TextQuestionAnswerTask extends StreamingAiTask {
|
|
|
6402
5911
|
var textQuestionAnswer = (input, config) => {
|
|
6403
5912
|
return new TextQuestionAnswerTask(config).run(input);
|
|
6404
5913
|
};
|
|
6405
|
-
|
|
5914
|
+
Workflow36.prototype.textQuestionAnswer = CreateWorkflow36(TextQuestionAnswerTask);
|
|
6406
5915
|
|
|
6407
5916
|
// src/task/TextRewriterTask.ts
|
|
6408
|
-
import { CreateWorkflow as
|
|
5917
|
+
import { CreateWorkflow as CreateWorkflow37, Workflow as Workflow37 } from "@workglow/task-graph";
|
|
6409
5918
|
var modelSchema27 = TypeModel("model:TextRewriterTask");
|
|
6410
5919
|
var TextRewriterInputSchema = {
|
|
6411
5920
|
type: "object",
|
|
@@ -6454,10 +5963,10 @@ class TextRewriterTask extends StreamingAiTask {
|
|
|
6454
5963
|
var textRewriter = (input, config) => {
|
|
6455
5964
|
return new TextRewriterTask(config).run(input);
|
|
6456
5965
|
};
|
|
6457
|
-
|
|
5966
|
+
Workflow37.prototype.textRewriter = CreateWorkflow37(TextRewriterTask);
|
|
6458
5967
|
|
|
6459
5968
|
// src/task/TextTranslationTask.ts
|
|
6460
|
-
import { CreateWorkflow as
|
|
5969
|
+
import { CreateWorkflow as CreateWorkflow38, Workflow as Workflow38 } from "@workglow/task-graph";
|
|
6461
5970
|
var modelSchema28 = TypeModel("model:TextTranslationTask");
|
|
6462
5971
|
var translationTextSchema = {
|
|
6463
5972
|
type: "string",
|
|
@@ -6520,10 +6029,10 @@ class TextTranslationTask extends StreamingAiTask {
|
|
|
6520
6029
|
var textTranslation = (input, config) => {
|
|
6521
6030
|
return new TextTranslationTask(config).run(input);
|
|
6522
6031
|
};
|
|
6523
|
-
|
|
6032
|
+
Workflow38.prototype.textTranslation = CreateWorkflow38(TextTranslationTask);
|
|
6524
6033
|
|
|
6525
6034
|
// src/task/ToolCallingTask.ts
|
|
6526
|
-
import { CreateWorkflow as
|
|
6035
|
+
import { CreateWorkflow as CreateWorkflow39, getTaskConstructors, Workflow as Workflow39 } from "@workglow/task-graph";
|
|
6527
6036
|
import { makeFingerprint } from "@workglow/util";
|
|
6528
6037
|
function taskTypesToTools(taskNames, registry) {
|
|
6529
6038
|
const constructors = getTaskConstructors(registry);
|
|
@@ -6755,16 +6264,16 @@ class ToolCallingTask extends StreamingAiTask {
|
|
|
6755
6264
|
var toolCalling = (input, config) => {
|
|
6756
6265
|
return new ToolCallingTask(config).run(input);
|
|
6757
6266
|
};
|
|
6758
|
-
|
|
6267
|
+
Workflow39.prototype.toolCalling = CreateWorkflow39(ToolCallingTask);
|
|
6759
6268
|
|
|
6760
6269
|
// src/task/TopicSegmenterTask.ts
|
|
6761
|
-
import { CreateWorkflow as
|
|
6270
|
+
import { CreateWorkflow as CreateWorkflow40, Task as Task15, Workflow as Workflow40 } from "@workglow/task-graph";
|
|
6762
6271
|
var SegmentationMethod = {
|
|
6763
6272
|
HEURISTIC: "heuristic",
|
|
6764
6273
|
EMBEDDING_SIMILARITY: "embedding-similarity",
|
|
6765
6274
|
HYBRID: "hybrid"
|
|
6766
6275
|
};
|
|
6767
|
-
var
|
|
6276
|
+
var inputSchema13 = {
|
|
6768
6277
|
type: "object",
|
|
6769
6278
|
properties: {
|
|
6770
6279
|
text: {
|
|
@@ -6805,7 +6314,7 @@ var inputSchema16 = {
|
|
|
6805
6314
|
required: ["text"],
|
|
6806
6315
|
additionalProperties: false
|
|
6807
6316
|
};
|
|
6808
|
-
var
|
|
6317
|
+
var outputSchema13 = {
|
|
6809
6318
|
type: "object",
|
|
6810
6319
|
properties: {
|
|
6811
6320
|
segments: {
|
|
@@ -6833,7 +6342,7 @@ var outputSchema16 = {
|
|
|
6833
6342
|
additionalProperties: false
|
|
6834
6343
|
};
|
|
6835
6344
|
|
|
6836
|
-
class TopicSegmenterTask extends
|
|
6345
|
+
class TopicSegmenterTask extends Task15 {
|
|
6837
6346
|
static type = "TopicSegmenterTask";
|
|
6838
6347
|
static category = "Document";
|
|
6839
6348
|
static title = "Topic Segmenter";
|
|
@@ -6841,10 +6350,10 @@ class TopicSegmenterTask extends Task18 {
|
|
|
6841
6350
|
static cacheable = true;
|
|
6842
6351
|
static EMBEDDING_DIMENSIONS = 256;
|
|
6843
6352
|
static inputSchema() {
|
|
6844
|
-
return
|
|
6353
|
+
return inputSchema13;
|
|
6845
6354
|
}
|
|
6846
6355
|
static outputSchema() {
|
|
6847
|
-
return
|
|
6356
|
+
return outputSchema13;
|
|
6848
6357
|
}
|
|
6849
6358
|
async execute(input, context) {
|
|
6850
6359
|
const {
|
|
@@ -7038,10 +6547,10 @@ class TopicSegmenterTask extends Task18 {
|
|
|
7038
6547
|
var topicSegmenter = (input, config) => {
|
|
7039
6548
|
return new TopicSegmenterTask(config).run(input);
|
|
7040
6549
|
};
|
|
7041
|
-
|
|
6550
|
+
Workflow40.prototype.topicSegmenter = CreateWorkflow40(TopicSegmenterTask);
|
|
7042
6551
|
|
|
7043
6552
|
// src/task/UnloadModelTask.ts
|
|
7044
|
-
import { CreateWorkflow as
|
|
6553
|
+
import { CreateWorkflow as CreateWorkflow41, Workflow as Workflow41 } from "@workglow/task-graph";
|
|
7045
6554
|
var modelSchema30 = TypeModel("model");
|
|
7046
6555
|
var UnloadModelInputSchema = {
|
|
7047
6556
|
type: "object",
|
|
@@ -7076,27 +6585,27 @@ class UnloadModelTask extends AiTask {
|
|
|
7076
6585
|
var unloadModel = (input, config) => {
|
|
7077
6586
|
return new UnloadModelTask(config).run(input);
|
|
7078
6587
|
};
|
|
7079
|
-
|
|
6588
|
+
Workflow41.prototype.unloadModel = CreateWorkflow41(UnloadModelTask);
|
|
7080
6589
|
|
|
7081
6590
|
// src/task/VectorQuantizeTask.ts
|
|
7082
|
-
import { CreateWorkflow as
|
|
6591
|
+
import { CreateWorkflow as CreateWorkflow42, Task as Task16, Workflow as Workflow42 } from "@workglow/task-graph";
|
|
7083
6592
|
import {
|
|
7084
6593
|
normalizeNumberArray,
|
|
7085
6594
|
TensorType,
|
|
7086
|
-
TypedArraySchema as
|
|
6595
|
+
TypedArraySchema as TypedArraySchema5
|
|
7087
6596
|
} from "@workglow/util/schema";
|
|
7088
|
-
var
|
|
6597
|
+
var inputSchema14 = {
|
|
7089
6598
|
type: "object",
|
|
7090
6599
|
properties: {
|
|
7091
6600
|
vector: {
|
|
7092
6601
|
anyOf: [
|
|
7093
|
-
|
|
6602
|
+
TypedArraySchema5({
|
|
7094
6603
|
title: "Vector",
|
|
7095
6604
|
description: "The vector to quantize"
|
|
7096
6605
|
}),
|
|
7097
6606
|
{
|
|
7098
6607
|
type: "array",
|
|
7099
|
-
items:
|
|
6608
|
+
items: TypedArraySchema5({
|
|
7100
6609
|
title: "Vector",
|
|
7101
6610
|
description: "Vector to quantize"
|
|
7102
6611
|
})
|
|
@@ -7122,18 +6631,18 @@ var inputSchema17 = {
|
|
|
7122
6631
|
required: ["vector", "targetType"],
|
|
7123
6632
|
additionalProperties: false
|
|
7124
6633
|
};
|
|
7125
|
-
var
|
|
6634
|
+
var outputSchema14 = {
|
|
7126
6635
|
type: "object",
|
|
7127
6636
|
properties: {
|
|
7128
6637
|
vector: {
|
|
7129
6638
|
anyOf: [
|
|
7130
|
-
|
|
6639
|
+
TypedArraySchema5({
|
|
7131
6640
|
title: "Quantized Vector",
|
|
7132
6641
|
description: "The quantized vector"
|
|
7133
6642
|
}),
|
|
7134
6643
|
{
|
|
7135
6644
|
type: "array",
|
|
7136
|
-
items:
|
|
6645
|
+
items: TypedArraySchema5({
|
|
7137
6646
|
title: "Quantized Vector",
|
|
7138
6647
|
description: "Quantized vector"
|
|
7139
6648
|
})
|
|
@@ -7159,19 +6668,22 @@ var outputSchema17 = {
|
|
|
7159
6668
|
additionalProperties: false
|
|
7160
6669
|
};
|
|
7161
6670
|
|
|
7162
|
-
class VectorQuantizeTask extends
|
|
6671
|
+
class VectorQuantizeTask extends Task16 {
|
|
7163
6672
|
static type = "VectorQuantizeTask";
|
|
7164
6673
|
static category = "Vector";
|
|
7165
6674
|
static title = "Quantize";
|
|
7166
6675
|
static description = "Quantize vectors to reduce storage and improve performance";
|
|
7167
6676
|
static cacheable = true;
|
|
7168
6677
|
static inputSchema() {
|
|
7169
|
-
return
|
|
6678
|
+
return inputSchema14;
|
|
7170
6679
|
}
|
|
7171
6680
|
static outputSchema() {
|
|
7172
|
-
return
|
|
6681
|
+
return outputSchema14;
|
|
6682
|
+
}
|
|
6683
|
+
async execute(input) {
|
|
6684
|
+
return this.executePreview(input);
|
|
7173
6685
|
}
|
|
7174
|
-
async
|
|
6686
|
+
async executePreview(input) {
|
|
7175
6687
|
const { vector, targetType, normalize = true } = input;
|
|
7176
6688
|
const isArray = Array.isArray(vector);
|
|
7177
6689
|
const vectors = isArray ? vector : [vector];
|
|
@@ -7259,15 +6771,15 @@ class VectorQuantizeTask extends Task19 {
|
|
|
7259
6771
|
var vectorQuantize = (input, config) => {
|
|
7260
6772
|
return new VectorQuantizeTask(config).run(input);
|
|
7261
6773
|
};
|
|
7262
|
-
|
|
6774
|
+
Workflow42.prototype.vectorQuantize = CreateWorkflow42(VectorQuantizeTask);
|
|
7263
6775
|
|
|
7264
6776
|
// src/task/VectorSimilarityTask.ts
|
|
7265
|
-
import { CreateWorkflow as
|
|
6777
|
+
import { CreateWorkflow as CreateWorkflow43, GraphAsTask, Workflow as Workflow43 } from "@workglow/task-graph";
|
|
7266
6778
|
import {
|
|
7267
6779
|
cosineSimilarity,
|
|
7268
6780
|
hammingSimilarity,
|
|
7269
6781
|
jaccardSimilarity,
|
|
7270
|
-
TypedArraySchema as
|
|
6782
|
+
TypedArraySchema as TypedArraySchema6
|
|
7271
6783
|
} from "@workglow/util/schema";
|
|
7272
6784
|
var SimilarityFn = {
|
|
7273
6785
|
COSINE: "cosine",
|
|
@@ -7282,13 +6794,13 @@ var similarityFunctions = {
|
|
|
7282
6794
|
var SimilarityInputSchema = {
|
|
7283
6795
|
type: "object",
|
|
7284
6796
|
properties: {
|
|
7285
|
-
query:
|
|
6797
|
+
query: TypedArraySchema6({
|
|
7286
6798
|
title: "Query",
|
|
7287
6799
|
description: "Query vector to compare against"
|
|
7288
6800
|
}),
|
|
7289
6801
|
vectors: {
|
|
7290
6802
|
type: "array",
|
|
7291
|
-
items:
|
|
6803
|
+
items: TypedArraySchema6({
|
|
7292
6804
|
title: "Input",
|
|
7293
6805
|
description: "Array of vectors to compare against the query"
|
|
7294
6806
|
})
|
|
@@ -7316,7 +6828,7 @@ var SimilarityOutputSchema = {
|
|
|
7316
6828
|
properties: {
|
|
7317
6829
|
output: {
|
|
7318
6830
|
type: "array",
|
|
7319
|
-
items:
|
|
6831
|
+
items: TypedArraySchema6({
|
|
7320
6832
|
title: "Output",
|
|
7321
6833
|
description: "Ranked output vectors"
|
|
7322
6834
|
})
|
|
@@ -7346,7 +6858,15 @@ class VectorSimilarityTask extends GraphAsTask {
|
|
|
7346
6858
|
static outputSchema() {
|
|
7347
6859
|
return SimilarityOutputSchema;
|
|
7348
6860
|
}
|
|
7349
|
-
async
|
|
6861
|
+
async execute(input) {
|
|
6862
|
+
return this.executePreview(input);
|
|
6863
|
+
}
|
|
6864
|
+
async executePreview({
|
|
6865
|
+
query,
|
|
6866
|
+
vectors,
|
|
6867
|
+
method,
|
|
6868
|
+
topK
|
|
6869
|
+
}) {
|
|
7350
6870
|
let similarities = [];
|
|
7351
6871
|
const fnName = method;
|
|
7352
6872
|
const fn = similarityFunctions[fnName];
|
|
@@ -7368,7 +6888,7 @@ class VectorSimilarityTask extends GraphAsTask {
|
|
|
7368
6888
|
var similarity = (input, config) => {
|
|
7369
6889
|
return new VectorSimilarityTask(config).run(input);
|
|
7370
6890
|
};
|
|
7371
|
-
|
|
6891
|
+
Workflow43.prototype.similarity = CreateWorkflow43(VectorSimilarityTask);
|
|
7372
6892
|
// src/task/MessageConversion.ts
|
|
7373
6893
|
function getInputMessages(input) {
|
|
7374
6894
|
const messages = input.messages;
|
|
@@ -7522,14 +7042,11 @@ var registerAiTasks = () => {
|
|
|
7522
7042
|
const tasks = [
|
|
7523
7043
|
AiChatTask,
|
|
7524
7044
|
BackgroundRemovalTask,
|
|
7525
|
-
ChunkToVectorTask,
|
|
7526
7045
|
CountTokensTask,
|
|
7527
7046
|
ContextBuilderTask,
|
|
7528
7047
|
DocumentEnricherTask,
|
|
7529
7048
|
DocumentUpsertTask,
|
|
7530
7049
|
ChunkRetrievalTask,
|
|
7531
|
-
ChunkVectorHybridSearchTask,
|
|
7532
|
-
ChunkVectorSearchTask,
|
|
7533
7050
|
ChunkVectorUpsertTask,
|
|
7534
7051
|
DownloadModelTask,
|
|
7535
7052
|
FaceDetectorTask,
|
|
@@ -7572,7 +7089,6 @@ var registerAiTasks = () => {
|
|
|
7572
7089
|
return tasks;
|
|
7573
7090
|
};
|
|
7574
7091
|
export {
|
|
7575
|
-
vectorStoreSearch,
|
|
7576
7092
|
vectorQuantize,
|
|
7577
7093
|
unloadModel,
|
|
7578
7094
|
topicSegmenter,
|
|
@@ -7614,7 +7130,6 @@ export {
|
|
|
7614
7130
|
imageSegmentation,
|
|
7615
7131
|
imageEmbedding,
|
|
7616
7132
|
imageClassification,
|
|
7617
|
-
hybridSearch,
|
|
7618
7133
|
hierarchyJoin,
|
|
7619
7134
|
hierarchicalChunker,
|
|
7620
7135
|
handLandmarker,
|
|
@@ -7630,7 +7145,6 @@ export {
|
|
|
7630
7145
|
countTokens,
|
|
7631
7146
|
contextBuilder,
|
|
7632
7147
|
chunkVectorUpsert,
|
|
7633
|
-
chunkToVector,
|
|
7634
7148
|
chunkRetrieval,
|
|
7635
7149
|
buildToolDescription,
|
|
7636
7150
|
backgroundRemoval,
|
|
@@ -7749,9 +7263,6 @@ export {
|
|
|
7749
7263
|
ContentBlockSchema,
|
|
7750
7264
|
ChunkingStrategy,
|
|
7751
7265
|
ChunkVectorUpsertTask,
|
|
7752
|
-
ChunkVectorSearchTask,
|
|
7753
|
-
ChunkVectorHybridSearchTask,
|
|
7754
|
-
ChunkToVectorTask,
|
|
7755
7266
|
ChunkRetrievalTask,
|
|
7756
7267
|
ChatMessageSchema,
|
|
7757
7268
|
BackgroundRemovalTask,
|
|
@@ -7766,4 +7277,4 @@ export {
|
|
|
7766
7277
|
AiChatInputSchema
|
|
7767
7278
|
};
|
|
7768
7279
|
|
|
7769
|
-
//# debugId=
|
|
7280
|
+
//# debugId=CCC95F44BED9528464756E2164756E21
|