@stackfactor/agent-utils 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/errorHandling.d.ts.map +1 -1
- package/dist/cjs/errorHandling.js +10 -5
- package/dist/cjs/index.d.ts +3 -0
- package/dist/cjs/index.d.ts.map +1 -1
- package/dist/cjs/index.js +3 -1
- package/dist/cjs/langChain.d.ts +1 -1
- package/dist/cjs/langChain.d.ts.map +1 -1
- package/dist/cjs/langChain.js +27 -8
- package/dist/cjs/stackfactorKnowledge.d.ts +182 -0
- package/dist/cjs/stackfactorKnowledge.d.ts.map +1 -0
- package/dist/cjs/stackfactorKnowledge.js +274 -0
- package/dist/esm/errorHandling.d.ts.map +1 -1
- package/dist/esm/errorHandling.js +10 -5
- package/dist/esm/index.d.ts +3 -0
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +2 -0
- package/dist/esm/langChain.d.ts +1 -1
- package/dist/esm/langChain.d.ts.map +1 -1
- package/dist/esm/langChain.js +27 -8
- package/dist/esm/stackfactorKnowledge.d.ts +182 -0
- package/dist/esm/stackfactorKnowledge.d.ts.map +1 -0
- package/dist/esm/stackfactorKnowledge.js +266 -0
- package/package.json +6 -5
|
@@ -81,11 +81,16 @@ const parseError = (error) => {
|
|
|
81
81
|
// A JSON-encoded error string, or just a plain message.
|
|
82
82
|
try {
|
|
83
83
|
const parsed = JSON.parse(error);
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
84
|
+
// Only an object is an encoded error; "404" or "true" also parse as
|
|
85
|
+
// JSON but are just the message text.
|
|
86
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
87
|
+
return {
|
|
88
|
+
code: isHttpCode(parsed.code) ? parsed.code : undefined,
|
|
89
|
+
message: parsed.message,
|
|
90
|
+
integration: parsed.integration,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
return { message: error };
|
|
89
94
|
}
|
|
90
95
|
catch {
|
|
91
96
|
return { message: error };
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { serve, runOnce, isRunOnceExecution, EXECUTION_ENV } from "./serve.js";
|
|
|
6
6
|
import { callAgent, checkAgent } from "./client.js";
|
|
7
7
|
import * as runtimeContext from "./runtimeContext.js";
|
|
8
8
|
import { recordUsage, USAGE_UNITS } from "./usage.js";
|
|
9
|
+
import stackfactorKnowledge from "./stackfactorKnowledge.js";
|
|
9
10
|
export { constants };
|
|
10
11
|
export { errorHandling, AppError };
|
|
11
12
|
export type { ParsedError } from "./errorHandling.js";
|
|
@@ -20,4 +21,6 @@ export type { AgentExecuteRequest, CallAgentOptions, CheckAgentOptions, AgentHea
|
|
|
20
21
|
export { runtimeContext };
|
|
21
22
|
export { recordUsage, USAGE_UNITS };
|
|
22
23
|
export type { UsageEntry, UsageUnit } from "./usage.js";
|
|
24
|
+
export { stackfactorKnowledge };
|
|
25
|
+
export type { GroundingPassage, KnowledgeChunk, KnowledgeSearch, KnowledgeSource, SkippedSource, StackFactorReference, } from "./stackfactorKnowledge.js";
|
|
23
26
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/esm/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,cAAc,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,YAAY,CAAC;AACnC,OAAO,aAAa,EAAE,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,SAAS,MAAM,gBAAgB,CAAC;AACvC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,cAAc,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,oBAAoB,MAAM,2BAA2B,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAEnC,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,YAAY,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEnD,YAAY,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhE,OAAO,EAAE,MAAM,EAAE,CAAC;AAElB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,CAAC;AAErB,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,iBAAiB,EACjB,WAAW,GACZ,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,cAAc,EAAE,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;AAEpC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,OAAO,EAAE,oBAAoB,EAAE,CAAC;AAEhC,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,eAAe,EACf,aAAa,EACb,oBAAoB,GACrB,MAAM,2BAA2B,CAAC"}
|
package/dist/esm/index.js
CHANGED
|
@@ -6,6 +6,7 @@ import { serve, runOnce, isRunOnceExecution, EXECUTION_ENV } from "./serve.js";
|
|
|
6
6
|
import { callAgent, checkAgent } from "./client.js";
|
|
7
7
|
import * as runtimeContext from "./runtimeContext.js";
|
|
8
8
|
import { recordUsage, USAGE_UNITS } from "./usage.js";
|
|
9
|
+
import stackfactorKnowledge from "./stackfactorKnowledge.js";
|
|
9
10
|
export { constants };
|
|
10
11
|
export { errorHandling, AppError };
|
|
11
12
|
export { langChain };
|
|
@@ -15,3 +16,4 @@ export { callAgent };
|
|
|
15
16
|
export { checkAgent };
|
|
16
17
|
export { runtimeContext };
|
|
17
18
|
export { recordUsage, USAGE_UNITS };
|
|
19
|
+
export { stackfactorKnowledge };
|
package/dist/esm/langChain.d.ts
CHANGED
|
@@ -62,7 +62,7 @@ declare const _default: {
|
|
|
62
62
|
supportsNativeSchema: (modelName: string) => boolean;
|
|
63
63
|
checkIfAIProviderConfigured: (config: any) => void;
|
|
64
64
|
createAgent: (name: string, modelName: string, systemPrompt: string, tools: any[], responseFormat: any, config: any, usageTracker?: UsageTracker | null) => any;
|
|
65
|
-
runAgent: (agent: any, prompt: string, config: any, onProgress?: Function | null, usageTracker?: UsageTracker | null) => Promise<any>;
|
|
65
|
+
runAgent: (agent: any, prompt: string, config: any, onProgress?: Function | null, usageTracker?: UsageTracker | null, progress?: number) => Promise<any>;
|
|
66
66
|
runPromptWithModel: (modelName: string, config: any, prompt: any, onProgressReport: any, minPercent?: number, maxPercent?: number, expectsJsonResponse?: boolean, schema?: any, agentName?: string, tools?: any[], usageTracker?: UsageTracker | null) => Promise<any>;
|
|
67
67
|
runPromptWithModelForImageGeneration: (modelName: string, config: any, prompt: string, options?: any, usageTracker?: UsageTracker | null) => Promise<any>;
|
|
68
68
|
runPromptWithModelForVideoGeneration: (modelName: string, config: any, prompt: string, options?: any, usageTracker?: UsageTracker | null) => Promise<any>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"langChain.d.ts","sourceRoot":"","sources":["../../src/langChain.ts"],"names":[],"mappings":"AAoGA,KAAK,UAAU,GAAG;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IAKtB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE;QAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACvC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACtD,CAAC;
|
|
1
|
+
{"version":3,"file":"langChain.d.ts","sourceRoot":"","sources":["../../src/langChain.ts"],"names":[],"mappings":"AAoGA,KAAK,UAAU,GAAG;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IAKtB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE;QAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACvC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACtD,CAAC;AA0iBF;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,eAAO,MAAM,WAAW,GACtB,WAAW,MAAM,EACjB,QAAQ,GAAG,EACX,SAAQ,GAAU,KACjB,GAwHF,CAAC;;8CAkb+C,OAAO,UAAU,GAAG,KAAG,OAAO;oCAWxC,GAAG,KAAG,OAAO;0CAgBtC,MAAM,cACN,GAAG,SACR,GAAG,KACT,MAAM;2CA0CqC,GAAG,UAAU,GAAG,KAAG,MAAM;sCApxC9B,MAAM,KAAG,OAAO;0CA+FZ,GAAG,KAAG,IAAI;wBA8sB/C,MAAM,aACD,MAAM,gBACH,MAAM,SACb,GAAG,EAAE,kBACI,GAAG,UACX,GAAG,iBACG,YAAY,GAAG,IAAI,KAChC,GAAG;sBAsCG,GAAG,UACF,MAAM,UACN,GAAG,eACC,QAAQ,GAAG,IAAI,iBACb,YAAY,GAAG,IAAI,aACvB,MAAM,KACf,OAAO,CAAC,GAAG,CAAC;oCAghBF,MAAM,UACT,GAAG,UACH,GAAG,oBACO,GAAG,eACT,MAAM,eACN,MAAM,wBACG,OAAO,WACpB,GAAG,cACA,MAAM,UACV,GAAG,EAAE,iBACE,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC;sDA2sBF,MAAM,UACT,GAAG,UACH,MAAM,YACL,GAAG,iBACE,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC;sDAmTF,MAAM,UACT,GAAG,UACH,MAAM,YACL,GAAG,iBACE,YAAY,GAAG,IAAI,KAChC,OAAO,CAAC,GAAG,CAAC;0CAx6C8B,GAAG,KAAG,MAAM;uCAxyB9C,YAAY,GAAG,IAAI,GAAG,SAAS,aAC7B,MAAM,YACP,GAAG,WACJ,GAAG,YACF,QAAQ,GAAG,QAAQ,KAC5B,IAAI;kCA5CI,YAAY,GAAG,IAAI,GAAG,SAAS,aAC7B,MAAM,SACV,UAAU,GAAG,IAAI,YACd,GAAG,KACZ,IAAI;+CA0FI,YAAY,GAAG,IAAI,GAAG,SAAS,aAC7B,MAAM,kBACD,MAAM,YACZ,GAAG,KACZ,IAAI;uCAoBI,YAAY,GAAG,IAAI,GAAG,SAAS,aAC7B,MAAM,WACR,MAAM,YACL,GAAG,KACZ,IAAI;mCAyuBU,MAAM,mBACJ,MAAM,EAAE,KACxB,MAAM;;AA28CT,wBAkBE"}
|
package/dist/esm/langChain.js
CHANGED
|
@@ -327,8 +327,14 @@ const formatZodErrors = (zodError) => {
|
|
|
327
327
|
const path = err.path.length > 0 ? err.path.join(".") : "root";
|
|
328
328
|
// Provide more context based on error type
|
|
329
329
|
switch (err.code) {
|
|
330
|
-
case "invalid_type":
|
|
331
|
-
|
|
330
|
+
case "invalid_type": {
|
|
331
|
+
// zod v3 sets `received`; zod v4 names it only in the message
|
|
332
|
+
// ("Invalid input: expected string, received number").
|
|
333
|
+
const received = err.received ??
|
|
334
|
+
/received (\w+)/.exec(String(err.message ?? ""))?.[1] ??
|
|
335
|
+
"a different type";
|
|
336
|
+
return `Field "${path}": Expected ${err.expected}, but received ${received}`;
|
|
337
|
+
}
|
|
332
338
|
case "invalid_literal":
|
|
333
339
|
return `Field "${path}": Expected literal value ${JSON.stringify(err.expected)}, but received ${JSON.stringify(err.received)}`;
|
|
334
340
|
case "unrecognized_keys":
|
|
@@ -764,19 +770,30 @@ const createAgent = (name, modelName, systemPrompt, tools = [], responseFormat,
|
|
|
764
770
|
* @param onProgress - Optional callback invoked with progress event objects throughout
|
|
765
771
|
* agent execution
|
|
766
772
|
* @param usageTracker - Optional usage accumulator updated on successful completion
|
|
773
|
+
* @param progress - The percentage every progress event reports (the stage's
|
|
774
|
+
* current one; the agent loop has no percentage of its own). Defaults to 0.
|
|
767
775
|
* @returns The raw response object returned by the agent's `invoke` method
|
|
768
776
|
*/
|
|
769
|
-
const runAgent = async (agent, prompt, config, onProgress = null, usageTracker = null) => {
|
|
777
|
+
const runAgent = async (agent, prompt, config, onProgress = null, usageTracker = null, progress = 0) => {
|
|
770
778
|
const startTime = Date.now();
|
|
779
|
+
// Every event carries `progress`: callers forward these to the platform's
|
|
780
|
+
// `{ progress, message }` progress bar, where a missing value would read as
|
|
781
|
+
// 0% and make the bar jump back during a tool call. The agent loop reports
|
|
782
|
+
// no percentage of its own, so the stage's current one is repeated.
|
|
771
783
|
const callbacks = onProgress
|
|
772
784
|
? [
|
|
773
785
|
{
|
|
774
|
-
handleToolStart
|
|
775
|
-
|
|
786
|
+
// LangChain calls handleToolStart(serializedTool, input, runId,
|
|
787
|
+
// parentRunId, tags, metadata, runName): the serialized tool
|
|
788
|
+
// ({ lc, type, id }) has no name; the tool's name is runName.
|
|
789
|
+
handleToolStart: (toolInfo, _input, _runId, _parentRunId, _tags, _metadata, runName) => {
|
|
790
|
+
const name = runName || toolInfo?.name || toolInfo?.kwargs?.name;
|
|
791
|
+
if (name)
|
|
776
792
|
onProgress({
|
|
777
793
|
type: "tool_start",
|
|
778
|
-
message: `Using "${
|
|
779
|
-
tool:
|
|
794
|
+
message: `Using "${name}" tool`,
|
|
795
|
+
tool: name,
|
|
796
|
+
progress,
|
|
780
797
|
});
|
|
781
798
|
},
|
|
782
799
|
handleAgentAction: (action) => {
|
|
@@ -785,12 +802,14 @@ const runAgent = async (agent, prompt, config, onProgress = null, usageTracker =
|
|
|
785
802
|
message: action.log || `Calling "${action.tool}" action`,
|
|
786
803
|
tool: action.tool,
|
|
787
804
|
input: action.toolInput,
|
|
805
|
+
progress,
|
|
788
806
|
});
|
|
789
807
|
},
|
|
790
808
|
handleChainError: (err) => {
|
|
791
809
|
onProgress({
|
|
792
810
|
type: "error",
|
|
793
811
|
message: `Error: ${err?.message || String(err)}`,
|
|
812
|
+
progress,
|
|
794
813
|
});
|
|
795
814
|
},
|
|
796
815
|
},
|
|
@@ -1254,7 +1273,7 @@ const runPromptWithModel = async (modelName, config, prompt, onProgressReport, m
|
|
|
1254
1273
|
let attempt = 0;
|
|
1255
1274
|
for (;;) {
|
|
1256
1275
|
// Run the agent with progress callback
|
|
1257
|
-
const response = await runAgent(agent, activeUserPrompt, config, onProgressReport || null, usageTracker);
|
|
1276
|
+
const response = await runAgent(agent, activeUserPrompt, config, onProgressReport || null, usageTracker, minPercent);
|
|
1258
1277
|
// Extract content from agent response
|
|
1259
1278
|
const messages = response?.messages || [];
|
|
1260
1279
|
if (messages.length === 0) {
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
/** What a STACKFACTOR reference to a knowledge base document carries (backend-core REFERENCE). */
|
|
2
|
+
export declare const STACKFACTOR_REFERENCE: Readonly<{
|
|
3
|
+
KIND: "STACKFACTOR";
|
|
4
|
+
TARGET_ELEMENT_TYPE: "KNOWLEDGE_BASE_DOCUMENT";
|
|
5
|
+
ORIGIN: "KB_EXTRACTION";
|
|
6
|
+
VISIBILITY: "EDIT";
|
|
7
|
+
}>;
|
|
8
|
+
/** How many documents one search may be kept to (the backend's bound); more are split. */
|
|
9
|
+
export declare const SEARCH_DOCUMENT_IDS_MAX = 500;
|
|
10
|
+
/** How long a query may be (the backend's bound); longer ones are cut. */
|
|
11
|
+
export declare const QUERY_MAX_LENGTH = 4000;
|
|
12
|
+
/** How many passages one search asks for when the caller does not say. */
|
|
13
|
+
export declare const DEFAULT_TOP_K = 8;
|
|
14
|
+
/** A chunk as the platform's search answers it (client-api `RetrievedChunk`). */
|
|
15
|
+
export interface KnowledgeChunk {
|
|
16
|
+
_id: string;
|
|
17
|
+
knowledgeBaseId?: string;
|
|
18
|
+
documentId: string;
|
|
19
|
+
documentName?: string | null;
|
|
20
|
+
sectionTitle?: string | null;
|
|
21
|
+
pageNumber?: number | null;
|
|
22
|
+
text?: string;
|
|
23
|
+
skills?: {
|
|
24
|
+
name: string | null;
|
|
25
|
+
microSkills?: string[];
|
|
26
|
+
}[];
|
|
27
|
+
score?: number | null;
|
|
28
|
+
[field: string]: unknown;
|
|
29
|
+
}
|
|
30
|
+
/** The platform's search, as client-api's `knowledgeBaseAgent.search` is called. */
|
|
31
|
+
export type KnowledgeSearch = (knowledgeBaseId: string, query: string, options: {
|
|
32
|
+
topK?: number;
|
|
33
|
+
documentIds?: string[];
|
|
34
|
+
}, token: string) => Promise<{
|
|
35
|
+
chunks: KnowledgeChunk[];
|
|
36
|
+
}>;
|
|
37
|
+
/** Where to search: a whole knowledge base, or only some of its documents. */
|
|
38
|
+
export interface KnowledgeSource {
|
|
39
|
+
knowledgeBaseId: string;
|
|
40
|
+
/** Only these documents; omitted, the whole knowledge base. */
|
|
41
|
+
documentIds?: string[];
|
|
42
|
+
}
|
|
43
|
+
/** A passage handed to the model, tagged so it can cite it. */
|
|
44
|
+
export interface GroundingPassage {
|
|
45
|
+
/** "k1", "k2"… in the order they rank. */
|
|
46
|
+
tag: string;
|
|
47
|
+
chunkId: string;
|
|
48
|
+
knowledgeBaseId: string;
|
|
49
|
+
documentId: string;
|
|
50
|
+
documentName: string | null;
|
|
51
|
+
sectionTitle: string | null;
|
|
52
|
+
pageNumber: number | null;
|
|
53
|
+
text: string;
|
|
54
|
+
/** The skills its passage teaches, as labelling named them. */
|
|
55
|
+
skills: {
|
|
56
|
+
name: string | null;
|
|
57
|
+
microSkills: string[];
|
|
58
|
+
}[];
|
|
59
|
+
score: number | null;
|
|
60
|
+
}
|
|
61
|
+
/** A knowledge base that could not be searched, and why. */
|
|
62
|
+
export interface SkippedSource {
|
|
63
|
+
knowledgeBaseId: string;
|
|
64
|
+
/** The HTTP status, when the platform answered one. */
|
|
65
|
+
status: number | null;
|
|
66
|
+
/** True when the person may not read it (or it is gone): expected, not a fault. */
|
|
67
|
+
noAccess: boolean;
|
|
68
|
+
message: string;
|
|
69
|
+
}
|
|
70
|
+
/** A STACKFACTOR reference to the knowledge base document a passage came from. */
|
|
71
|
+
export interface StackFactorReference {
|
|
72
|
+
kind: "STACKFACTOR";
|
|
73
|
+
title: string;
|
|
74
|
+
description: string | null;
|
|
75
|
+
target: {
|
|
76
|
+
elementType: "KNOWLEDGE_BASE_DOCUMENT";
|
|
77
|
+
elementId: string;
|
|
78
|
+
parentId: string;
|
|
79
|
+
};
|
|
80
|
+
knowledgeBase: {
|
|
81
|
+
knowledgeBaseId: string;
|
|
82
|
+
documentId: string;
|
|
83
|
+
chunkId: string;
|
|
84
|
+
sectionTitle: string | null;
|
|
85
|
+
};
|
|
86
|
+
origin: "KB_EXTRACTION";
|
|
87
|
+
visibility: "EDIT";
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* The knowledge bases, and the documents in them, that a list of references
|
|
91
|
+
* cites: its STACKFACTOR references to knowledge base documents, by their
|
|
92
|
+
* `knowledgeBase` ids, or by their `target` when only that is there. Any other
|
|
93
|
+
* reference is passed over. Each knowledge base once, its documents once each,
|
|
94
|
+
* in the order they first appear.
|
|
95
|
+
* @param references A skill's (or a microskill's) references
|
|
96
|
+
* @returns The sources to search
|
|
97
|
+
*/
|
|
98
|
+
export declare const sourcesFromReferences: (references: unknown) => KnowledgeSource[];
|
|
99
|
+
/**
|
|
100
|
+
* The knowledge bases skill coverage found (client-api
|
|
101
|
+
* `knowledgeBase.getSkillCoverage`): every one the person can open that holds
|
|
102
|
+
* the skill, each searched whole.
|
|
103
|
+
* @param coverage The coverage answer, `{ knowledgeBases: [{ _id }] }`
|
|
104
|
+
* @returns The sources to search
|
|
105
|
+
*/
|
|
106
|
+
export declare const sourcesFromCoverage: (coverage: unknown) => KnowledgeSource[];
|
|
107
|
+
/**
|
|
108
|
+
* Searches every source for every query, under the person's access, and hands
|
|
109
|
+
* back the passages found, best first, each once, tagged "k1", "k2"… for the
|
|
110
|
+
* model to cite. A source with documents is searched within them (in batches of
|
|
111
|
+
* SEARCH_DOCUMENT_IDS_MAX); one without, whole. A source the person may not
|
|
112
|
+
* read (401, 403, 404) is skipped: none of its passages are used and no more of
|
|
113
|
+
* its searches are made. A search that fails otherwise is reported and the
|
|
114
|
+
* passages its other searches found stand. Either way it is in `skipped`, once
|
|
115
|
+
* per knowledge base, and the passages of the others stand.
|
|
116
|
+
* @param options `sources`, `queries`, `search` (client-api
|
|
117
|
+
* `knowledgeBaseAgent.search`), `token` (the run's AGENT token), `topK`
|
|
118
|
+
* per search (default DEFAULT_TOP_K)
|
|
119
|
+
* @returns `{ passages, skipped }`
|
|
120
|
+
*/
|
|
121
|
+
export declare const retrievePassages: (options: {
|
|
122
|
+
sources: KnowledgeSource[];
|
|
123
|
+
queries: string[];
|
|
124
|
+
search: KnowledgeSearch;
|
|
125
|
+
token: string;
|
|
126
|
+
topK?: number;
|
|
127
|
+
}) => Promise<{
|
|
128
|
+
passages: GroundingPassage[];
|
|
129
|
+
skipped: SkippedSource[];
|
|
130
|
+
}>;
|
|
131
|
+
/**
|
|
132
|
+
* The passages as a prompt shows them: tag, where each came from, and its text.
|
|
133
|
+
* @param passages From `retrievePassages`
|
|
134
|
+
* @returns `[{ tag, document, section, text }]`, for the model's input
|
|
135
|
+
*/
|
|
136
|
+
export declare const passagesForPrompt: (passages: GroundingPassage[]) => {
|
|
137
|
+
tag: string;
|
|
138
|
+
document: string | null;
|
|
139
|
+
section: string | null;
|
|
140
|
+
text: string;
|
|
141
|
+
}[];
|
|
142
|
+
/**
|
|
143
|
+
* STACKFACTOR references for passages: one per passage, in their order, with no
|
|
144
|
+
* cap. With `tags`, only the passages the model cited by tag (any tag it made
|
|
145
|
+
* up is passed over); without, every passage.
|
|
146
|
+
* @param passages From `retrievePassages`
|
|
147
|
+
* @param tags The tags the model cited ("k3", "K3" and " k3 " alike)
|
|
148
|
+
* @returns The references, ready for a skill's, a microskill's or a question's `references`
|
|
149
|
+
*/
|
|
150
|
+
export declare const referencesFor: (passages: GroundingPassage[], tags?: unknown[]) => StackFactorReference[];
|
|
151
|
+
declare const _default: {
|
|
152
|
+
DEFAULT_TOP_K: number;
|
|
153
|
+
QUERY_MAX_LENGTH: number;
|
|
154
|
+
SEARCH_DOCUMENT_IDS_MAX: number;
|
|
155
|
+
STACKFACTOR_REFERENCE: Readonly<{
|
|
156
|
+
KIND: "STACKFACTOR";
|
|
157
|
+
TARGET_ELEMENT_TYPE: "KNOWLEDGE_BASE_DOCUMENT";
|
|
158
|
+
ORIGIN: "KB_EXTRACTION";
|
|
159
|
+
VISIBILITY: "EDIT";
|
|
160
|
+
}>;
|
|
161
|
+
passagesForPrompt: (passages: GroundingPassage[]) => {
|
|
162
|
+
tag: string;
|
|
163
|
+
document: string | null;
|
|
164
|
+
section: string | null;
|
|
165
|
+
text: string;
|
|
166
|
+
}[];
|
|
167
|
+
referencesFor: (passages: GroundingPassage[], tags?: unknown[]) => StackFactorReference[];
|
|
168
|
+
retrievePassages: (options: {
|
|
169
|
+
sources: KnowledgeSource[];
|
|
170
|
+
queries: string[];
|
|
171
|
+
search: KnowledgeSearch;
|
|
172
|
+
token: string;
|
|
173
|
+
topK?: number;
|
|
174
|
+
}) => Promise<{
|
|
175
|
+
passages: GroundingPassage[];
|
|
176
|
+
skipped: SkippedSource[];
|
|
177
|
+
}>;
|
|
178
|
+
sourcesFromCoverage: (coverage: unknown) => KnowledgeSource[];
|
|
179
|
+
sourcesFromReferences: (references: unknown) => KnowledgeSource[];
|
|
180
|
+
};
|
|
181
|
+
export default _default;
|
|
182
|
+
//# sourceMappingURL=stackfactorKnowledge.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stackfactorKnowledge.d.ts","sourceRoot":"","sources":["../../src/stackfactorKnowledge.ts"],"names":[],"mappings":"AA8BA,kGAAkG;AAClG,eAAO,MAAM,qBAAqB;;;;;EAKhC,CAAC;AAEH,0FAA0F;AAC1F,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C,0EAA0E;AAC1E,eAAO,MAAM,gBAAgB,OAAO,CAAC;AAErC,0EAA0E;AAC1E,eAAO,MAAM,aAAa,IAAI,CAAC;AAQ/B,iFAAiF;AACjF,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IAC3D,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,oFAAoF;AACpF,MAAM,MAAM,eAAe,GAAG,CAC5B,eAAe,EAAE,MAAM,EACvB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,EAClD,KAAK,EAAE,MAAM,KACV,OAAO,CAAC;IAAE,MAAM,EAAE,cAAc,EAAE,CAAA;CAAE,CAAC,CAAC;AAE3C,8EAA8E;AAC9E,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,+DAA+D;AAC/D,MAAM,WAAW,gBAAgB;IAC/B,0CAA0C;IAC1C,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IACzD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,4DAA4D;AAC5D,MAAM,WAAW,aAAa;IAC5B,eAAe,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,mFAAmF;IACnF,QAAQ,EAAE,OAAO,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,kFAAkF;AAClF,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,aAAa,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE;QAAE,WAAW,EAAE,yBAAyB,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IACxF,aAAa,EAAE;QACb,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;IACF,MAAM,EAAE,eAAe,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD;;;;;;;;GAQG;AACH,eAAO,MAAM,qBAAqB,GAAI,YAAY,OAAO,KAAG,eAAe,EAmB1E,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,GAAI,UAAU,OAAO,KAAG,eAAe,EAWtE,CAAC;AA8BF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,gBAAgB,GAAU,SAAS;IAC9C,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,KAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE,CAgFrE,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,GAC5B,UAAU,gBAAgB,EAAE,KAC3B;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAM5E,CAAC;AAEN;;;;;;;GAOG;AACH,eAAO,MAAM,aAAa,GACxB,UAAU,gBAAgB,EAAE,EAC5B,OAAO,OAAO,EAAE,KACf,oBAAoB,EAyBtB,CAAC;;;;;;;;;;;kCA7CU,gBAAgB,EAAE,KAC3B;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE;8BAiBvE,gBAAgB,EAAE,SACrB,OAAO,EAAE,KACf,oBAAoB,EAAE;gCAlHuB;QAC9C,OAAO,EAAE,eAAe,EAAE,CAAC;QAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,MAAM,EAAE,eAAe,CAAC;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,KAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;QAAC,OAAO,EAAE,aAAa,EAAE,CAAA;KAAE,CAAC;oCA7DzB,OAAO,KAAG,eAAe,EAAE;wCA5BvB,OAAO,KAAG,eAAe,EAAE;;AAgO7E,wBAUE"}
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
//
|
|
2
|
+
// StackFactor knowledge — grounding generated content in the knowledge bases
|
|
3
|
+
//
|
|
4
|
+
// A generation agent grounded in StackFactor (the Additional Information data
|
|
5
|
+
// source "StackFactor", `dataSource: "RAG"`) writes from knowledge-base
|
|
6
|
+
// passages instead of a web search, and cites every passage it used as a
|
|
7
|
+
// STACKFACTOR reference. A later agent generating for the same skill (its
|
|
8
|
+
// assessments, its learning content) reads those references and searches the
|
|
9
|
+
// documents they cite. Designed with Candin, 2026-10-08:
|
|
10
|
+
//
|
|
11
|
+
// - which knowledge bases: every one the person can open that holds the
|
|
12
|
+
// skill (skill coverage), never a fixed number of them;
|
|
13
|
+
// - which references: one per passage used, with no cap per document or per
|
|
14
|
+
// microskill;
|
|
15
|
+
// - access: the platform's search judges the person the run acts for, so a
|
|
16
|
+
// run reads only what that person may open. A knowledge base the person
|
|
17
|
+
// cannot read is skipped and reported, never an error: the agent
|
|
18
|
+
// generates without it.
|
|
19
|
+
//
|
|
20
|
+
// A reference points at the content and never copies it: the passage is
|
|
21
|
+
// searched again, under the reader's own access, whenever it is used. It is
|
|
22
|
+
// EDIT-only (`visibility: "EDIT"`): the platform strips it from what is
|
|
23
|
+
// published, so a knowledge base's titles never reach a learner; agents read
|
|
24
|
+
// it from the draft.
|
|
25
|
+
//
|
|
26
|
+
// The search itself is the caller's (`search`: client-api
|
|
27
|
+
// `knowledgeBaseAgent.search`, 1.11 or later for `documentIds`), so this
|
|
28
|
+
// module needs no particular client-api.
|
|
29
|
+
//
|
|
30
|
+
/** What a STACKFACTOR reference to a knowledge base document carries (backend-core REFERENCE). */
|
|
31
|
+
export const STACKFACTOR_REFERENCE = Object.freeze({
|
|
32
|
+
KIND: "STACKFACTOR",
|
|
33
|
+
TARGET_ELEMENT_TYPE: "KNOWLEDGE_BASE_DOCUMENT",
|
|
34
|
+
ORIGIN: "KB_EXTRACTION",
|
|
35
|
+
VISIBILITY: "EDIT",
|
|
36
|
+
});
|
|
37
|
+
/** How many documents one search may be kept to (the backend's bound); more are split. */
|
|
38
|
+
export const SEARCH_DOCUMENT_IDS_MAX = 500;
|
|
39
|
+
/** How long a query may be (the backend's bound); longer ones are cut. */
|
|
40
|
+
export const QUERY_MAX_LENGTH = 4000;
|
|
41
|
+
/** How many passages one search asks for when the caller does not say. */
|
|
42
|
+
export const DEFAULT_TOP_K = 8;
|
|
43
|
+
/** How many searches run at once. */
|
|
44
|
+
const CONCURRENCY = 4;
|
|
45
|
+
/** Statuses that mean the person may not read the knowledge base, or it is gone. */
|
|
46
|
+
const NO_ACCESS = new Set([401, 403, 404]);
|
|
47
|
+
const text = (value) => (typeof value === "string" ? value.trim() : "");
|
|
48
|
+
const id = (value) => typeof value === "string" ? value.trim() : value != null && typeof value === "object" ? String(value) : "";
|
|
49
|
+
/**
|
|
50
|
+
* The knowledge bases, and the documents in them, that a list of references
|
|
51
|
+
* cites: its STACKFACTOR references to knowledge base documents, by their
|
|
52
|
+
* `knowledgeBase` ids, or by their `target` when only that is there. Any other
|
|
53
|
+
* reference is passed over. Each knowledge base once, its documents once each,
|
|
54
|
+
* in the order they first appear.
|
|
55
|
+
* @param references A skill's (or a microskill's) references
|
|
56
|
+
* @returns The sources to search
|
|
57
|
+
*/
|
|
58
|
+
export const sourcesFromReferences = (references) => {
|
|
59
|
+
const byBase = new Map();
|
|
60
|
+
for (const reference of Array.isArray(references) ? references : []) {
|
|
61
|
+
if (reference?.kind !== STACKFACTOR_REFERENCE.KIND)
|
|
62
|
+
continue;
|
|
63
|
+
const target = reference.target ?? {};
|
|
64
|
+
const isDocumentTarget = target.elementType === STACKFACTOR_REFERENCE.TARGET_ELEMENT_TYPE;
|
|
65
|
+
const knowledgeBaseId = id(reference.knowledgeBase?.knowledgeBaseId) || (isDocumentTarget ? id(target.parentId) : "");
|
|
66
|
+
const documentId = id(reference.knowledgeBase?.documentId) || (isDocumentTarget ? id(target.elementId) : "");
|
|
67
|
+
if (!knowledgeBaseId || !documentId)
|
|
68
|
+
continue;
|
|
69
|
+
const documents = byBase.get(knowledgeBaseId) ?? new Set();
|
|
70
|
+
documents.add(documentId);
|
|
71
|
+
byBase.set(knowledgeBaseId, documents);
|
|
72
|
+
}
|
|
73
|
+
return [...byBase.entries()].map(([knowledgeBaseId, documents]) => ({
|
|
74
|
+
knowledgeBaseId,
|
|
75
|
+
documentIds: [...documents],
|
|
76
|
+
}));
|
|
77
|
+
};
|
|
78
|
+
/**
|
|
79
|
+
* The knowledge bases skill coverage found (client-api
|
|
80
|
+
* `knowledgeBase.getSkillCoverage`): every one the person can open that holds
|
|
81
|
+
* the skill, each searched whole.
|
|
82
|
+
* @param coverage The coverage answer, `{ knowledgeBases: [{ _id }] }`
|
|
83
|
+
* @returns The sources to search
|
|
84
|
+
*/
|
|
85
|
+
export const sourcesFromCoverage = (coverage) => {
|
|
86
|
+
const bases = coverage?.knowledgeBases;
|
|
87
|
+
const seen = new Set();
|
|
88
|
+
const sources = [];
|
|
89
|
+
for (const base of Array.isArray(bases) ? bases : []) {
|
|
90
|
+
const knowledgeBaseId = id(base?._id);
|
|
91
|
+
if (!knowledgeBaseId || seen.has(knowledgeBaseId))
|
|
92
|
+
continue;
|
|
93
|
+
seen.add(knowledgeBaseId);
|
|
94
|
+
sources.push({ knowledgeBaseId });
|
|
95
|
+
}
|
|
96
|
+
return sources;
|
|
97
|
+
};
|
|
98
|
+
/** The documents of a source in batches the backend accepts; one batch of none for a whole knowledge base. */
|
|
99
|
+
const batchesOf = (source) => {
|
|
100
|
+
if (!Array.isArray(source.documentIds))
|
|
101
|
+
return [undefined];
|
|
102
|
+
const batches = [];
|
|
103
|
+
for (let at = 0; at < source.documentIds.length; at += SEARCH_DOCUMENT_IDS_MAX) {
|
|
104
|
+
batches.push(source.documentIds.slice(at, at + SEARCH_DOCUMENT_IDS_MAX));
|
|
105
|
+
}
|
|
106
|
+
return batches;
|
|
107
|
+
};
|
|
108
|
+
/** Runs `worker` over `items`, at most `limit` at a time. */
|
|
109
|
+
const inPool = async (items, limit, worker) => {
|
|
110
|
+
let next = 0;
|
|
111
|
+
const run = async () => {
|
|
112
|
+
while (next < items.length) {
|
|
113
|
+
const index = next++;
|
|
114
|
+
await worker(items[index]);
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, run));
|
|
118
|
+
};
|
|
119
|
+
/** The status of a failed platform call, from an axios error or one shaped like it. */
|
|
120
|
+
const statusOf = (error) => {
|
|
121
|
+
const status = error?.response?.status ?? error?.status;
|
|
122
|
+
return typeof status === "number" ? status : null;
|
|
123
|
+
};
|
|
124
|
+
/**
|
|
125
|
+
* Searches every source for every query, under the person's access, and hands
|
|
126
|
+
* back the passages found, best first, each once, tagged "k1", "k2"… for the
|
|
127
|
+
* model to cite. A source with documents is searched within them (in batches of
|
|
128
|
+
* SEARCH_DOCUMENT_IDS_MAX); one without, whole. A source the person may not
|
|
129
|
+
* read (401, 403, 404) is skipped: none of its passages are used and no more of
|
|
130
|
+
* its searches are made. A search that fails otherwise is reported and the
|
|
131
|
+
* passages its other searches found stand. Either way it is in `skipped`, once
|
|
132
|
+
* per knowledge base, and the passages of the others stand.
|
|
133
|
+
* @param options `sources`, `queries`, `search` (client-api
|
|
134
|
+
* `knowledgeBaseAgent.search`), `token` (the run's AGENT token), `topK`
|
|
135
|
+
* per search (default DEFAULT_TOP_K)
|
|
136
|
+
* @returns `{ passages, skipped }`
|
|
137
|
+
*/
|
|
138
|
+
export const retrievePassages = async (options) => {
|
|
139
|
+
const queries = [
|
|
140
|
+
...new Set((Array.isArray(options.queries) ? options.queries : [])
|
|
141
|
+
.map((query) => text(query).slice(0, QUERY_MAX_LENGTH))
|
|
142
|
+
.filter(Boolean)),
|
|
143
|
+
];
|
|
144
|
+
const sources = Array.isArray(options.sources) ? options.sources : [];
|
|
145
|
+
const topK = options.topK ?? DEFAULT_TOP_K;
|
|
146
|
+
const searches = [];
|
|
147
|
+
for (const source of sources) {
|
|
148
|
+
for (const documentIds of batchesOf(source)) {
|
|
149
|
+
if (documentIds && documentIds.length === 0)
|
|
150
|
+
continue;
|
|
151
|
+
for (const query of queries) {
|
|
152
|
+
searches.push({ source, query, documentIds, order: searches.length });
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
const found = new Map();
|
|
157
|
+
const skipped = new Map();
|
|
158
|
+
await inPool(searches, CONCURRENCY, async ({ source, query, documentIds, order }) => {
|
|
159
|
+
if (skipped.get(source.knowledgeBaseId)?.noAccess)
|
|
160
|
+
return;
|
|
161
|
+
try {
|
|
162
|
+
const answer = await options.search(source.knowledgeBaseId, query, documentIds ? { topK, documentIds } : { topK }, options.token);
|
|
163
|
+
(Array.isArray(answer?.chunks) ? answer.chunks : []).forEach((chunk, rank) => {
|
|
164
|
+
const chunkId = id(chunk?._id);
|
|
165
|
+
if (!chunkId || !id(chunk?.documentId))
|
|
166
|
+
return;
|
|
167
|
+
const known = found.get(chunkId);
|
|
168
|
+
const better = !known || (chunk.score ?? 0) > (known.chunk.score ?? 0);
|
|
169
|
+
// Ties keep the first search's rank, so the order is the searches' own.
|
|
170
|
+
found.set(chunkId, {
|
|
171
|
+
chunk: better ? chunk : known.chunk,
|
|
172
|
+
knowledgeBaseId: source.knowledgeBaseId,
|
|
173
|
+
order: known ? Math.min(known.order, order * 1000 + rank) : order * 1000 + rank,
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
catch (error) {
|
|
178
|
+
const status = statusOf(error);
|
|
179
|
+
const noAccess = status !== null && NO_ACCESS.has(status);
|
|
180
|
+
// A refusal outranks a failure reported before it.
|
|
181
|
+
if (skipped.has(source.knowledgeBaseId) && !noAccess)
|
|
182
|
+
return;
|
|
183
|
+
skipped.set(source.knowledgeBaseId, {
|
|
184
|
+
knowledgeBaseId: source.knowledgeBaseId,
|
|
185
|
+
message: typeof error?.message === "string" ? error.message : String(error),
|
|
186
|
+
noAccess,
|
|
187
|
+
status,
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
const passages = [...found.entries()]
|
|
192
|
+
.filter(([, entry]) => !skipped.get(entry.knowledgeBaseId)?.noAccess)
|
|
193
|
+
.sort(([, a], [, b]) => (b.chunk.score ?? 0) - (a.chunk.score ?? 0) || a.order - b.order)
|
|
194
|
+
.map(([chunkId, { chunk, knowledgeBaseId }], index) => ({
|
|
195
|
+
chunkId,
|
|
196
|
+
documentId: id(chunk.documentId),
|
|
197
|
+
documentName: text(chunk.documentName) || null,
|
|
198
|
+
knowledgeBaseId: id(chunk.knowledgeBaseId) || knowledgeBaseId,
|
|
199
|
+
pageNumber: typeof chunk.pageNumber === "number" ? chunk.pageNumber : null,
|
|
200
|
+
score: typeof chunk.score === "number" ? chunk.score : null,
|
|
201
|
+
sectionTitle: text(chunk.sectionTitle) || null,
|
|
202
|
+
skills: (Array.isArray(chunk.skills) ? chunk.skills : []).map((skill) => ({
|
|
203
|
+
microSkills: (Array.isArray(skill?.microSkills) ? skill.microSkills : []).filter((name) => typeof name === "string"),
|
|
204
|
+
name: typeof skill?.name === "string" ? skill.name : null,
|
|
205
|
+
})),
|
|
206
|
+
tag: `k${index + 1}`,
|
|
207
|
+
text: typeof chunk.text === "string" ? chunk.text : "",
|
|
208
|
+
}));
|
|
209
|
+
return { passages, skipped: [...skipped.values()] };
|
|
210
|
+
};
|
|
211
|
+
/**
|
|
212
|
+
* The passages as a prompt shows them: tag, where each came from, and its text.
|
|
213
|
+
* @param passages From `retrievePassages`
|
|
214
|
+
* @returns `[{ tag, document, section, text }]`, for the model's input
|
|
215
|
+
*/
|
|
216
|
+
export const passagesForPrompt = (passages) => (Array.isArray(passages) ? passages : []).map((passage) => ({
|
|
217
|
+
document: passage.documentName,
|
|
218
|
+
section: passage.sectionTitle,
|
|
219
|
+
tag: passage.tag,
|
|
220
|
+
text: passage.text,
|
|
221
|
+
}));
|
|
222
|
+
/**
|
|
223
|
+
* STACKFACTOR references for passages: one per passage, in their order, with no
|
|
224
|
+
* cap. With `tags`, only the passages the model cited by tag (any tag it made
|
|
225
|
+
* up is passed over); without, every passage.
|
|
226
|
+
* @param passages From `retrievePassages`
|
|
227
|
+
* @param tags The tags the model cited ("k3", "K3" and " k3 " alike)
|
|
228
|
+
* @returns The references, ready for a skill's, a microskill's or a question's `references`
|
|
229
|
+
*/
|
|
230
|
+
export const referencesFor = (passages, tags) => {
|
|
231
|
+
const list = Array.isArray(passages) ? passages : [];
|
|
232
|
+
const cited = Array.isArray(tags)
|
|
233
|
+
? new Set(tags.map((tag) => text(tag).toLowerCase()).filter(Boolean))
|
|
234
|
+
: null;
|
|
235
|
+
return list
|
|
236
|
+
.filter((passage) => !cited || cited.has(passage.tag.toLowerCase()))
|
|
237
|
+
.map((passage) => ({
|
|
238
|
+
description: passage.sectionTitle,
|
|
239
|
+
kind: STACKFACTOR_REFERENCE.KIND,
|
|
240
|
+
knowledgeBase: {
|
|
241
|
+
chunkId: passage.chunkId,
|
|
242
|
+
documentId: passage.documentId,
|
|
243
|
+
knowledgeBaseId: passage.knowledgeBaseId,
|
|
244
|
+
sectionTitle: passage.sectionTitle,
|
|
245
|
+
},
|
|
246
|
+
origin: STACKFACTOR_REFERENCE.ORIGIN,
|
|
247
|
+
target: {
|
|
248
|
+
elementId: passage.documentId,
|
|
249
|
+
elementType: STACKFACTOR_REFERENCE.TARGET_ELEMENT_TYPE,
|
|
250
|
+
parentId: passage.knowledgeBaseId,
|
|
251
|
+
},
|
|
252
|
+
title: passage.documentName || passage.sectionTitle || "Knowledge base document",
|
|
253
|
+
visibility: STACKFACTOR_REFERENCE.VISIBILITY,
|
|
254
|
+
}));
|
|
255
|
+
};
|
|
256
|
+
export default {
|
|
257
|
+
DEFAULT_TOP_K,
|
|
258
|
+
QUERY_MAX_LENGTH,
|
|
259
|
+
SEARCH_DOCUMENT_IDS_MAX,
|
|
260
|
+
STACKFACTOR_REFERENCE,
|
|
261
|
+
passagesForPrompt,
|
|
262
|
+
referencesFor,
|
|
263
|
+
retrievePassages,
|
|
264
|
+
sourcesFromCoverage,
|
|
265
|
+
sourcesFromReferences,
|
|
266
|
+
};
|
package/package.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"publishConfig": {
|
|
4
4
|
"access": "public"
|
|
5
5
|
},
|
|
6
|
-
"version": "1.
|
|
6
|
+
"version": "1.8.0",
|
|
7
7
|
"description": "",
|
|
8
8
|
"main": "dist/cjs/index.js",
|
|
9
9
|
"module": "dist/esm/index.js",
|
|
@@ -20,8 +20,9 @@
|
|
|
20
20
|
"work:release": "bash scripts/release.sh",
|
|
21
21
|
"release": "bash scripts/release.sh",
|
|
22
22
|
"release:fanout": "bash scripts/release-fanout.sh",
|
|
23
|
-
"ar-auth": "npm config set @stackfactor:registry https://us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/; TOKEN=$(gcloud auth print-access-token 2>/dev/null || curl -sfH 'Metadata-Flavor: Google' http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token | sed -n 's/.*\"access_token\":\"\\([^\"]*\\)\".*/\\1/p'); test -n \"$TOKEN\" || { echo 'ar-auth: no credential
|
|
24
|
-
"reinstall": "npm run ar-auth && npm ci"
|
|
23
|
+
"ar-auth": "npm config set @stackfactor:registry https://us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/; TOKEN=$(gcloud auth print-access-token 2>/dev/null || curl -sfH 'Metadata-Flavor: Google' http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token | sed -n 's/.*\"access_token\":\"\\([^\"]*\\)\".*/\\1/p'); test -n \"$TOKEN\" || { echo 'ar-auth: no credential \u2014 run: gcloud auth login' >&2; exit 1; }; npm config set //us-central1-npm.pkg.dev/virtual-development-team/sf-devenv-npm-private/:_authToken \"$TOKEN\"; npm config set //us-central1-npm.pkg.dev/virtual-development-team/:_authToken \"$TOKEN\"; echo 'ar-auth: Artifact Registry scope + token configured'",
|
|
24
|
+
"reinstall": "npm run ar-auth && npm ci",
|
|
25
|
+
"test:coverage": "npm run build && node --experimental-test-coverage --test --test-reporter=tap test/*.test.js"
|
|
25
26
|
},
|
|
26
27
|
"repository": {
|
|
27
28
|
"type": "git",
|
|
@@ -49,7 +50,7 @@
|
|
|
49
50
|
"dependencies": {
|
|
50
51
|
"@anthropic-ai/sdk": "^0.74.0",
|
|
51
52
|
"@google/genai": "^1.46.0",
|
|
52
|
-
"@grpc/grpc-js": "^1.14.
|
|
53
|
+
"@grpc/grpc-js": "^1.14.5",
|
|
53
54
|
"@grpc/proto-loader": "^0.7.15",
|
|
54
55
|
"@langchain/anthropic": "^1.3.25",
|
|
55
56
|
"@langchain/core": "^1.2.9",
|
|
@@ -58,7 +59,7 @@
|
|
|
58
59
|
"@opentelemetry/api": "^1.9.0",
|
|
59
60
|
"@stackfactor/client-api": "^1.6.0",
|
|
60
61
|
"langchain": "^1.2.35",
|
|
61
|
-
"mongoose": "^9.
|
|
62
|
+
"mongoose": "^9.10.4",
|
|
62
63
|
"openai": "^6.32.0",
|
|
63
64
|
"winston": "^3.19.0",
|
|
64
65
|
"zod": "^4.3.6"
|