@inflexa-ai/harness 0.46.0 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/agents/conversation-agent.d.ts +3 -8
- package/dist/agents/conversation-agent.js +24 -4
- package/dist/agents/report-session-agent.d.ts +11 -14
- package/dist/agents/report-session-agent.js +30 -17
- package/dist/agents/sandbox/shared.d.ts +18 -13
- package/dist/agents/sandbox/shared.js +42 -27
- package/dist/agents/sandbox/types.d.ts +1 -1
- package/dist/agents/sandbox-catalog.d.ts +2 -0
- package/dist/agents/sandbox-catalog.js +30 -1
- package/dist/app/chat-turn-workflow.js +5 -6
- package/dist/app/synthesize-run.d.ts +10 -6
- package/dist/app/synthesize-run.js +4 -2
- package/dist/contracts/analogy-report.d.ts +46 -2
- package/dist/contracts/analogy-report.js +31 -25
- package/dist/contracts/chat-parts.d.ts +3 -14
- package/dist/contracts/report-reference.d.ts +2 -1
- package/dist/contracts/report-reference.js +2 -1
- package/dist/contracts/schemas/chat-parts.d.ts +0 -27
- package/dist/contracts/schemas/chat-parts.js +1 -3
- package/dist/execution/run-canceler.d.ts +2 -0
- package/dist/execution/run-canceler.js +9 -0
- package/dist/execution/run-synthesis.d.ts +36 -8
- package/dist/execution/run-synthesis.js +114 -32
- package/dist/index.d.ts +2 -5
- package/dist/index.js +3 -6
- package/dist/input-scan/enrich.d.ts +5 -10
- package/dist/input-scan/enrich.js +12 -8
- package/dist/lib/hooks.d.ts +9 -0
- package/dist/lib/hooks.js +6 -1
- package/dist/lib/markdown-pdf.d.ts +27 -0
- package/dist/lib/markdown-pdf.js +236 -0
- package/dist/lib/metrics.d.ts +31 -0
- package/dist/lib/metrics.js +64 -0
- package/dist/lib/pdf-text.d.ts +23 -0
- package/dist/lib/pdf-text.js +34 -0
- package/dist/lib/regulatory-corpus/fetch-fda-guidance.js +3 -5
- package/dist/lib/regulatory-corpus/fetch-ich.js +3 -5
- package/dist/loop/run-loop.d.ts +5 -0
- package/dist/loop/run-loop.js +30 -2
- package/dist/loop/sub-agent.js +5 -6
- package/dist/loop/tool-name-repair.d.ts +11 -0
- package/dist/loop/tool-name-repair.js +23 -0
- package/dist/loop/turn-credentials.d.ts +5 -0
- package/dist/loop/turn-credentials.js +17 -0
- package/dist/loop/types.d.ts +14 -0
- package/dist/memory/card-builders.d.ts +1 -1
- package/dist/prompts/SOUL.d.ts +6 -6
- package/dist/prompts/SOUL.js +22 -121
- package/dist/prompts/analogical-reasoner.d.ts +1 -1
- package/dist/prompts/analogical-reasoner.js +35 -101
- package/dist/prompts/briefing.d.ts +20 -11
- package/dist/prompts/briefing.js +36 -19
- package/dist/prompts/conversation.d.ts +1 -1
- package/dist/prompts/conversation.js +103 -383
- package/dist/prompts/planner.d.ts +1 -5
- package/dist/prompts/planner.js +45 -128
- package/dist/prompts/report-session.d.ts +1 -1
- package/dist/prompts/report-session.js +26 -57
- package/dist/prompts/sandbox/bulk-transcriptomics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/bulk-transcriptomics-agent.js +3 -3
- package/dist/prompts/sandbox/cheminformatics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/cheminformatics-agent.js +12 -47
- package/dist/prompts/sandbox/chromatin-agent.d.ts +1 -1
- package/dist/prompts/sandbox/chromatin-agent.js +4 -5
- package/dist/prompts/sandbox/data-profiler.js +11 -34
- package/dist/prompts/sandbox/dna-methylation-agent.d.ts +1 -1
- package/dist/prompts/sandbox/dna-methylation-agent.js +3 -1
- package/dist/prompts/sandbox/drug-repurposing-agent.d.ts +1 -1
- package/dist/prompts/sandbox/drug-repurposing-agent.js +23 -73
- package/dist/prompts/sandbox/enrichment-agent.d.ts +1 -1
- package/dist/prompts/sandbox/enrichment-agent.js +3 -1
- package/dist/prompts/sandbox/genomic-variant-agent.d.ts +1 -1
- package/dist/prompts/sandbox/genomic-variant-agent.js +3 -4
- package/dist/prompts/sandbox/immune-profiling-agent.d.ts +1 -1
- package/dist/prompts/sandbox/immune-profiling-agent.js +5 -4
- package/dist/prompts/sandbox/metabolomics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/metabolomics-agent.js +3 -1
- package/dist/prompts/sandbox/microbiome-agent.d.ts +1 -1
- package/dist/prompts/sandbox/microbiome-agent.js +3 -1
- package/dist/prompts/sandbox/multi-omics-integration-agent.d.ts +1 -1
- package/dist/prompts/sandbox/multi-omics-integration-agent.js +3 -1
- package/dist/prompts/sandbox/multimodal-sc-agent.d.ts +1 -1
- package/dist/prompts/sandbox/multimodal-sc-agent.js +3 -1
- package/dist/prompts/sandbox/network-agent.d.ts +1 -1
- package/dist/prompts/sandbox/network-agent.js +3 -1
- package/dist/prompts/sandbox/pkpd-clinical-response-agent.d.ts +1 -1
- package/dist/prompts/sandbox/pkpd-clinical-response-agent.js +4 -7
- package/dist/prompts/sandbox/proteomics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/proteomics-agent.js +3 -1
- package/dist/prompts/sandbox/scientific-executor.d.ts +1 -1
- package/dist/prompts/sandbox/scientific-executor.js +0 -12
- package/dist/prompts/sandbox/single-cell-agent.d.ts +1 -1
- package/dist/prompts/sandbox/single-cell-agent.js +3 -1
- package/dist/prompts/sandbox/spatial-omics-agent.d.ts +1 -1
- package/dist/prompts/sandbox/spatial-omics-agent.js +3 -1
- package/dist/prompts/sandbox/statistical-modeling-agent.d.ts +1 -1
- package/dist/prompts/sandbox/statistical-modeling-agent.js +4 -4
- package/dist/prompts/sandbox/translational-safety-agent.d.ts +1 -1
- package/dist/prompts/sandbox/translational-safety-agent.js +10 -29
- package/dist/prompts/sandbox-standards.d.ts +15 -18
- package/dist/prompts/sandbox-standards.js +96 -305
- package/dist/prompts/synthesis-agent.d.ts +1 -1
- package/dist/prompts/synthesis-agent.js +24 -90
- package/dist/provenance/collector.d.ts +2 -5
- package/dist/provenance/collector.js +2 -5
- package/dist/provenance/seam.d.ts +36 -1
- package/dist/provenance/workspace-reads.d.ts +36 -0
- package/dist/provenance/workspace-reads.js +33 -0
- package/dist/providers/request-headers.js +12 -3
- package/dist/report-model/production-resolver.d.ts +22 -52
- package/dist/report-model/production-resolver.js +43 -112
- package/dist/report-render/chart-export.d.ts +9 -0
- package/dist/report-render/chart-export.js +1 -1
- package/dist/report-render/views/chart-view.d.ts +1 -0
- package/dist/report-render/views/chart-view.js +1 -0
- package/dist/report-render/views/lineage.d.ts +1 -0
- package/dist/report-render/views/page-view.d.ts +1 -0
- package/dist/report-render/views/page-view.js +1 -0
- package/dist/report-render/views/prose.d.ts +1 -0
- package/dist/report-render/views/prose.js +1 -0
- package/dist/report-render/views/references-view.d.ts +1 -0
- package/dist/report-render/views/values.d.ts +1 -0
- package/dist/report-render/views/values.js +1 -0
- package/dist/runtime/assemble.d.ts +20 -9
- package/dist/runtime/assemble.js +47 -38
- package/dist/sandbox/client.d.ts +5 -3
- package/dist/sandbox/create-sandbox.d.ts +7 -12
- package/dist/sandbox/create-sandbox.js +15 -37
- package/dist/sandbox/docker-client.d.ts +1 -3
- package/dist/sandbox/docker-client.js +3 -5
- package/dist/sandbox/exec-outcome.d.ts +11 -11
- package/dist/sandbox/exec-outcome.js +22 -22
- package/dist/sandbox/exec.d.ts +5 -0
- package/dist/sandbox/exec.js +9 -1
- package/dist/sandbox/identity.d.ts +4 -16
- package/dist/sandbox/identity.js +5 -21
- package/dist/sandbox/k8s-client.d.ts +0 -1
- package/dist/sandbox/k8s-client.js +8 -13
- package/dist/sandbox/lease-names.d.ts +6 -0
- package/dist/sandbox/lease-names.js +12 -0
- package/dist/sandbox/leases.d.ts +105 -0
- package/dist/sandbox/leases.js +519 -0
- package/dist/sandbox/liveness.d.ts +5 -2
- package/dist/sandbox/mount-plan.d.ts +10 -1
- package/dist/sandbox/mount-plan.js +7 -1
- package/dist/sandbox/reconciler.d.ts +58 -0
- package/dist/sandbox/reconciler.js +153 -0
- package/dist/sandbox/sandbox-step-translate.d.ts +1 -1
- package/dist/sandbox/types.d.ts +25 -18
- package/dist/sandbox/types.js +10 -4
- package/dist/schemas/plan-schemas.d.ts +0 -21
- package/dist/schemas/plan-schemas.js +2 -3
- package/dist/schemas/run-synthesis.d.ts +2 -2
- package/dist/schemas/run-synthesis.js +3 -3
- package/dist/schemas/validate-plan.d.ts +1 -14
- package/dist/schemas/validate-plan.js +5 -24
- package/dist/schemas/workflow-state.d.ts +3 -32
- package/dist/schemas/workflow-state.js +4 -13
- package/dist/state/index.d.ts +0 -1
- package/dist/state/index.js +0 -1
- package/dist/state/migrations/20261030120000_lease_the_sandboxes.d.ts +7 -0
- package/dist/state/migrations/20261030120000_lease_the_sandboxes.js +34 -0
- package/dist/state/migrations/20261030120100_drop_the_report_derivations.d.ts +5 -0
- package/dist/state/migrations/20261030120100_drop_the_report_derivations.js +9 -0
- package/dist/state/migrations/20261030120200_keep_the_report_derivations.d.ts +6 -0
- package/dist/state/migrations/20261030120200_keep_the_report_derivations.js +10 -0
- package/dist/state/migrations/index.js +6 -0
- package/dist/state/purge-analysis.d.ts +4 -2
- package/dist/state/purge-analysis.js +22 -11
- package/dist/state/sandbox-leases.d.ts +81 -0
- package/dist/state/sandbox-leases.js +247 -0
- package/dist/state/schema.d.ts +1 -13
- package/dist/state/schema.js +1 -6
- package/dist/state/step-executions.d.ts +0 -2
- package/dist/state/step-executions.js +1 -4
- package/dist/tasks/data-profile.d.ts +40 -2
- package/dist/tasks/data-profile.js +89 -87
- package/dist/testing.d.ts +2 -0
- package/dist/testing.js +6 -1
- package/dist/tools/ad-hoc-router.d.ts +1 -6
- package/dist/tools/ad-hoc-router.js +5 -48
- package/dist/tools/bio/comptox.js +1 -1
- package/dist/tools/bio/drug-gene-interactions.js +1 -1
- package/dist/tools/bio/gene-disease-evidence.js +1 -1
- package/dist/tools/bio/keys.d.ts +1 -1
- package/dist/tools/bio/pubmed.js +31 -0
- package/dist/tools/bio/search-gene.js +17 -0
- package/dist/tools/bio/search-geo-datasets.js +13 -0
- package/dist/tools/bio/search-regulatory-approvals.js +1 -1
- package/dist/tools/define-tool.d.ts +25 -2
- package/dist/tools/define-tool.js +11 -1
- package/dist/tools/display/show-user.js +1 -1
- package/dist/tools/execute-analysis.d.ts +1 -3
- package/dist/tools/execute-analysis.js +5 -33
- package/dist/tools/report-session/index.d.ts +0 -1
- package/dist/tools/report-session/index.js +0 -1
- package/dist/tools/report-session/script-tools.d.ts +30 -0
- package/dist/tools/report-session/script-tools.js +177 -0
- package/dist/tools/research/generate-analogy-report.d.ts +16 -9
- package/dist/tools/research/generate-analogy-report.js +111 -49
- package/dist/tools/research/generate-plan.d.ts +102 -7
- package/dist/tools/research/generate-plan.js +74 -43
- package/dist/tools/research/index.d.ts +4 -3
- package/dist/tools/research/index.js +4 -3
- package/dist/tools/research/inspect-run.d.ts +0 -1
- package/dist/tools/research/inspect-run.js +5 -10
- package/dist/tools/research/search-arxiv.js +15 -0
- package/dist/tools/research/search-github-repos.js +13 -0
- package/dist/tools/research/search-semantic-scholar.js +16 -0
- package/dist/tools/research/synthesis-lookups.d.ts +21 -0
- package/dist/tools/research/synthesis-lookups.js +29 -0
- package/dist/tools/research/web-fetch.d.ts +1 -1
- package/dist/tools/research/web-fetch.js +14 -12
- package/dist/tools/sandbox/lease-tools.d.ts +32 -0
- package/dist/tools/sandbox/lease-tools.js +60 -0
- package/dist/tools/script/codemode-worker.d.ts +5 -0
- package/dist/tools/script/codemode-worker.js +5 -0
- package/dist/tools/script/file-tools.d.ts +100 -0
- package/dist/tools/script/file-tools.js +377 -0
- package/dist/tools/script/metrics.d.ts +21 -0
- package/dist/tools/script/metrics.js +38 -0
- package/dist/tools/script/quickjs.wasm +0 -0
- package/dist/tools/script/run-script.d.ts +76 -0
- package/dist/tools/script/run-script.js +326 -0
- package/dist/tools/script/scratch.d.ts +81 -0
- package/dist/tools/script/scratch.js +142 -0
- package/dist/tools/start-report-session.js +1 -1
- package/dist/tools/workspace/edit-file.js +2 -1
- package/dist/tools/workspace/execute-command.d.ts +9 -13
- package/dist/tools/workspace/execute-command.js +23 -19
- package/dist/tools/workspace/file-stat.js +1 -1
- package/dist/tools/workspace/grep.d.ts +5 -2
- package/dist/tools/workspace/grep.js +19 -10
- package/dist/tools/workspace/mutator.d.ts +48 -4
- package/dist/tools/workspace/mutator.js +43 -12
- package/dist/tools/workspace/read-file.d.ts +5 -2
- package/dist/tools/workspace/read-file.js +15 -11
- package/dist/tools/workspace/read-surface.d.ts +17 -0
- package/dist/tools/workspace/read-surface.js +17 -0
- package/dist/tools/workspace/result-bounds.d.ts +1 -0
- package/dist/tools/workspace/run-exec.d.ts +12 -13
- package/dist/tools/workspace/run-exec.js +13 -13
- package/dist/tools/workspace/scan-inputs.d.ts +2 -5
- package/dist/tools/workspace/scan-inputs.js +2 -3
- package/dist/tools/workspace/show-file.js +1 -1
- package/dist/tools/workspace/show-plan.js +4 -3
- package/dist/tools/workspace/write-file.js +2 -1
- package/dist/workflows/execute-analysis-scheduler.d.ts +0 -26
- package/dist/workflows/execute-analysis-scheduler.js +12 -32
- package/dist/workflows/execute-analysis.d.ts +17 -35
- package/dist/workflows/execute-analysis.js +13 -107
- package/dist/workflows/pause.js +29 -22
- package/dist/workflows/sandbox-step.d.ts +16 -31
- package/dist/workflows/sandbox-step.js +36 -71
- package/dist/workflows/test/sql-writes.d.ts +0 -1
- package/dist/workflows/test/sql-writes.js +0 -2
- package/dist/workspace/filesystem.d.ts +2 -2
- package/dist/workspace/filesystem.js +10 -5
- package/dist/workspace/paths.d.ts +19 -1
- package/dist/workspace/paths.js +22 -1
- package/package.json +6 -3
- package/dist/prompts/literature-reviewer.d.ts +0 -1
- package/dist/prompts/literature-reviewer.js +0 -114
- package/dist/sandbox/reaper.d.ts +0 -81
- package/dist/sandbox/reaper.js +0 -161
- package/dist/sandbox/step-sandbox.d.ts +0 -47
- package/dist/sandbox/step-sandbox.js +0 -53
- package/dist/state/active-sandboxes.d.ts +0 -33
- package/dist/state/active-sandboxes.js +0 -65
- package/dist/tasks/extract-values-script.d.ts +0 -64
- package/dist/tasks/extract-values-script.js +0 -180
- package/dist/tasks/extract-values.d.ts +0 -77
- package/dist/tasks/extract-values.js +0 -146
- package/dist/tools/report-session/derive-table.d.ts +0 -164
- package/dist/tools/report-session/derive-table.js +0 -419
- package/dist/tools/research/literature-reviewer.d.ts +0 -42
- package/dist/tools/research/literature-reviewer.js +0 -107
- package/dist/workflows/metrics.d.ts +0 -23
- package/dist/workflows/metrics.js +0 -38
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A PDF from markdown, on `marked` and `pdfmake`, with no browser.
|
|
3
|
+
*
|
|
4
|
+
* `marked` gives the token tree, and each token maps to a `pdfmake` node: a heading, a paragraph, a list, a GFM table,
|
|
5
|
+
* a code block, a quote, a rule, and an image. An SVG image, for example a chart of `render_chart`, lands as vector
|
|
6
|
+
* graphics. The text font is the Roboto that `pdfmake` ships, which holds the Greek letters and the common math signs.
|
|
7
|
+
*
|
|
8
|
+
* `pdfmake` resolves a font by a file path, also inside an SVG, thus the fonts land one time in a private temporary
|
|
9
|
+
* directory. Each URL is refused, and a local path is refused unless it is one of those font files, thus a document
|
|
10
|
+
* reads nothing that the caller did not give it.
|
|
11
|
+
*/
|
|
12
|
+
import { mkdtemp, writeFile } from "node:fs/promises";
|
|
13
|
+
import { tmpdir } from "node:os";
|
|
14
|
+
import { join } from "node:path";
|
|
15
|
+
import { lexer } from "marked";
|
|
16
|
+
import { z } from "zod";
|
|
17
|
+
/** The token kinds of the lexer of `marked`. */
|
|
18
|
+
const MARKED_KINDS = new Set([
|
|
19
|
+
"blockquote",
|
|
20
|
+
"br",
|
|
21
|
+
"checkbox",
|
|
22
|
+
"code",
|
|
23
|
+
"codespan",
|
|
24
|
+
"def",
|
|
25
|
+
"del",
|
|
26
|
+
"em",
|
|
27
|
+
"escape",
|
|
28
|
+
"heading",
|
|
29
|
+
"hr",
|
|
30
|
+
"html",
|
|
31
|
+
"image",
|
|
32
|
+
"link",
|
|
33
|
+
"list",
|
|
34
|
+
"list_item",
|
|
35
|
+
"paragraph",
|
|
36
|
+
"space",
|
|
37
|
+
"strong",
|
|
38
|
+
"table",
|
|
39
|
+
"tag",
|
|
40
|
+
"text",
|
|
41
|
+
]);
|
|
42
|
+
/**
|
|
43
|
+
* A token of one of the kinds of the lexer. No extension of `marked` is registered, thus a token with one of these
|
|
44
|
+
* kinds has the shape of that kind.
|
|
45
|
+
*/
|
|
46
|
+
function isMarkedToken(token) {
|
|
47
|
+
return MARKED_KINDS.has(token.type);
|
|
48
|
+
}
|
|
49
|
+
/** The tokens of the lexer as its own kinds. */
|
|
50
|
+
function tokensOf(tokens) {
|
|
51
|
+
return (tokens ?? []).filter(isMarkedToken);
|
|
52
|
+
}
|
|
53
|
+
const FontFilesSchema = z.record(z.string(), z.string());
|
|
54
|
+
/** The width of the text block of an A4 page with the margins of {@link documentOf}, in points. */
|
|
55
|
+
const TEXT_WIDTH = 515;
|
|
56
|
+
const FONT_FILES = {
|
|
57
|
+
normal: "Roboto-Regular.ttf",
|
|
58
|
+
bold: "Roboto-Medium.ttf",
|
|
59
|
+
italics: "Roboto-Italic.ttf",
|
|
60
|
+
bolditalics: "Roboto-MediumItalic.ttf",
|
|
61
|
+
};
|
|
62
|
+
let ready;
|
|
63
|
+
/** The `pdfmake` instance with the fonts on disk and the access policies set. It is set up one time for each process. */
|
|
64
|
+
function pdfmakeOf() {
|
|
65
|
+
ready ??= (async () => {
|
|
66
|
+
const { default: pdfmake } = await import("pdfmake");
|
|
67
|
+
const vfsModule = await import("pdfmake/build/vfs_fonts.js");
|
|
68
|
+
const vfs = FontFilesSchema.parse(Reflect.get(Object(vfsModule), "default"));
|
|
69
|
+
const dir = await mkdtemp(join(tmpdir(), "pdf-fonts-"));
|
|
70
|
+
const paths = {};
|
|
71
|
+
for (const [style, file] of Object.entries(FONT_FILES)) {
|
|
72
|
+
const data = vfs[file];
|
|
73
|
+
if (data === undefined)
|
|
74
|
+
throw new Error(`pdfmake ships no font file ${file}`);
|
|
75
|
+
paths[style] = join(dir, file);
|
|
76
|
+
await writeFile(join(dir, file), Buffer.from(data, "base64"));
|
|
77
|
+
}
|
|
78
|
+
const fontPaths = new Set(Object.values(paths));
|
|
79
|
+
pdfmake.setFonts({ Roboto: paths });
|
|
80
|
+
pdfmake.setUrlAccessPolicy(() => false);
|
|
81
|
+
pdfmake.setLocalAccessPolicy((path) => fontPaths.has(path));
|
|
82
|
+
return pdfmake;
|
|
83
|
+
})();
|
|
84
|
+
return ready;
|
|
85
|
+
}
|
|
86
|
+
/** The text runs of inline tokens, with the styles of the marks that hold them. */
|
|
87
|
+
function inline(tokens, images) {
|
|
88
|
+
const out = [];
|
|
89
|
+
for (const token of tokensOf(tokens)) {
|
|
90
|
+
switch (token.type) {
|
|
91
|
+
case "strong":
|
|
92
|
+
out.push({ text: inline(token.tokens, images), bold: true });
|
|
93
|
+
break;
|
|
94
|
+
case "em":
|
|
95
|
+
out.push({ text: inline(token.tokens, images), italics: true });
|
|
96
|
+
break;
|
|
97
|
+
case "del":
|
|
98
|
+
out.push({ text: inline(token.tokens, images), decoration: "lineThrough" });
|
|
99
|
+
break;
|
|
100
|
+
case "codespan":
|
|
101
|
+
out.push({ text: token.text, background: "#f2f2f2" });
|
|
102
|
+
break;
|
|
103
|
+
case "link":
|
|
104
|
+
out.push({ text: inline(token.tokens, images), link: token.href, color: "#1a5fb4", decoration: "underline" });
|
|
105
|
+
break;
|
|
106
|
+
case "br":
|
|
107
|
+
out.push({ text: "\n" });
|
|
108
|
+
break;
|
|
109
|
+
case "image":
|
|
110
|
+
out.push({ text: token.text, italics: true });
|
|
111
|
+
break;
|
|
112
|
+
case "text":
|
|
113
|
+
out.push(token.tokens === undefined ? { text: token.text } : { text: inline(token.tokens, images) });
|
|
114
|
+
break;
|
|
115
|
+
case "escape":
|
|
116
|
+
out.push({ text: token.text });
|
|
117
|
+
break;
|
|
118
|
+
default:
|
|
119
|
+
out.push({ text: token.raw });
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return out;
|
|
123
|
+
}
|
|
124
|
+
/** An image of a paragraph that holds one image alone, with its alt text as the caption. */
|
|
125
|
+
function imageNode(token, images) {
|
|
126
|
+
const image = images.get(token.href);
|
|
127
|
+
const caption = token.text.length > 0 ? [{ text: token.text, style: "caption" }] : [];
|
|
128
|
+
if (image === undefined)
|
|
129
|
+
return [{ text: `[${token.text || token.href}]`, italics: true }];
|
|
130
|
+
if (image.kind === "svg")
|
|
131
|
+
return [{ svg: image.svg, width: TEXT_WIDTH, margin: [0, 6, 0, 2] }, ...caption];
|
|
132
|
+
const data = `data:image/${image.kind};base64,${Buffer.from(image.bytes).toString("base64")}`;
|
|
133
|
+
return [{ image: data, fit: [TEXT_WIDTH, 600], margin: [0, 6, 0, 2] }, ...caption];
|
|
134
|
+
}
|
|
135
|
+
function blocks(tokens, images) {
|
|
136
|
+
const out = [];
|
|
137
|
+
for (const token of tokensOf(tokens)) {
|
|
138
|
+
switch (token.type) {
|
|
139
|
+
case "heading":
|
|
140
|
+
out.push({ text: inline(token.tokens, images), style: `h${String(Math.min(token.depth, 4))}` });
|
|
141
|
+
break;
|
|
142
|
+
case "paragraph": {
|
|
143
|
+
const parts = tokensOf(token.tokens);
|
|
144
|
+
const only = parts.length === 1 ? parts[0] : undefined;
|
|
145
|
+
if (only?.type === "image")
|
|
146
|
+
out.push(...imageNode(only, images));
|
|
147
|
+
else
|
|
148
|
+
out.push({ text: inline(token.tokens, images), style: "paragraph" });
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
case "list":
|
|
152
|
+
out.push({
|
|
153
|
+
[token.ordered ? "ol" : "ul"]: token.items.map((item) => ({ stack: blocks(item.tokens, images) })),
|
|
154
|
+
style: "list",
|
|
155
|
+
});
|
|
156
|
+
break;
|
|
157
|
+
case "table":
|
|
158
|
+
out.push({
|
|
159
|
+
table: {
|
|
160
|
+
headerRows: 1,
|
|
161
|
+
widths: token.header.map(() => "*"),
|
|
162
|
+
body: [
|
|
163
|
+
token.header.map((cell) => ({ text: inline(cell.tokens, images), bold: true })),
|
|
164
|
+
...token.rows.map((row) => row.map((cell) => ({ text: inline(cell.tokens, images) }))),
|
|
165
|
+
],
|
|
166
|
+
},
|
|
167
|
+
layout: "lightHorizontalLines",
|
|
168
|
+
style: "table",
|
|
169
|
+
});
|
|
170
|
+
break;
|
|
171
|
+
case "code":
|
|
172
|
+
out.push({ text: token.text, style: "code", preserveLeadingSpaces: true });
|
|
173
|
+
break;
|
|
174
|
+
case "blockquote":
|
|
175
|
+
out.push({ stack: blocks(token.tokens, images), style: "quote" });
|
|
176
|
+
break;
|
|
177
|
+
case "hr":
|
|
178
|
+
out.push({ canvas: [{ type: "line", x1: 0, y1: 0, x2: TEXT_WIDTH, y2: 0, lineWidth: 0.5, lineColor: "#999999" }], margin: [0, 8, 0, 8] });
|
|
179
|
+
break;
|
|
180
|
+
case "text":
|
|
181
|
+
out.push({ text: inline(token.tokens, images), style: "paragraph" });
|
|
182
|
+
break;
|
|
183
|
+
default:
|
|
184
|
+
// A space, an HTML block, or a link definition prints nothing.
|
|
185
|
+
break;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
return out;
|
|
189
|
+
}
|
|
190
|
+
/** The source of each image of a markdown text, in the order of the text. */
|
|
191
|
+
export function markdownImageSources(markdown) {
|
|
192
|
+
const sources = [];
|
|
193
|
+
const walk = (tokens) => {
|
|
194
|
+
for (const token of tokensOf(tokens)) {
|
|
195
|
+
if (token.type === "image")
|
|
196
|
+
sources.push(token.href);
|
|
197
|
+
if (token.type === "list")
|
|
198
|
+
for (const item of token.items)
|
|
199
|
+
walk(item.tokens);
|
|
200
|
+
else if (token.type === "table")
|
|
201
|
+
for (const cell of [...token.header, ...token.rows.flat()])
|
|
202
|
+
walk(cell.tokens);
|
|
203
|
+
else if ("tokens" in token)
|
|
204
|
+
walk(token.tokens);
|
|
205
|
+
}
|
|
206
|
+
};
|
|
207
|
+
walk(lexer(markdown));
|
|
208
|
+
return [...new Set(sources)];
|
|
209
|
+
}
|
|
210
|
+
function documentOf(markdown, title, images) {
|
|
211
|
+
return {
|
|
212
|
+
info: title === undefined ? {} : { title },
|
|
213
|
+
pageSize: "A4",
|
|
214
|
+
pageMargins: [40, 48, 40, 48],
|
|
215
|
+
content: [...(title === undefined ? [] : [{ text: title, style: "title" }]), ...blocks(lexer(markdown), images)],
|
|
216
|
+
defaultStyle: { font: "Roboto", fontSize: 10, lineHeight: 1.25 },
|
|
217
|
+
styles: {
|
|
218
|
+
title: { fontSize: 20, bold: true, margin: [0, 0, 0, 12] },
|
|
219
|
+
h1: { fontSize: 16, bold: true, margin: [0, 12, 0, 6] },
|
|
220
|
+
h2: { fontSize: 13, bold: true, margin: [0, 10, 0, 4] },
|
|
221
|
+
h3: { fontSize: 11, bold: true, margin: [0, 8, 0, 4] },
|
|
222
|
+
h4: { fontSize: 10, bold: true, margin: [0, 6, 0, 2] },
|
|
223
|
+
paragraph: { margin: [0, 0, 0, 6] },
|
|
224
|
+
list: { margin: [0, 0, 0, 6] },
|
|
225
|
+
table: { fontSize: 9, margin: [0, 4, 0, 8] },
|
|
226
|
+
code: { fontSize: 8.5, background: "#f2f2f2", margin: [0, 2, 0, 8] },
|
|
227
|
+
quote: { color: "#555555", italics: true, margin: [12, 0, 0, 6] },
|
|
228
|
+
caption: { fontSize: 8.5, italics: true, color: "#555555", alignment: "center", margin: [0, 0, 0, 8] },
|
|
229
|
+
},
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
/** The PDF of a markdown text. `images` holds each image that the text names, by its source. */
|
|
233
|
+
export async function markdownToPdf(args) {
|
|
234
|
+
const pdfmake = await pdfmakeOf();
|
|
235
|
+
return pdfmake.createPdf(documentOf(args.markdown, args.title, args.images)).getBuffer();
|
|
236
|
+
}
|
package/dist/lib/metrics.d.ts
CHANGED
|
@@ -10,6 +10,9 @@
|
|
|
10
10
|
* - cortex.sandbox.exec.duration{outcome} — histogram, the runtime the sandbox reported for that exec
|
|
11
11
|
* - cortex.sandbox.exec.peak_memory_bytes{outcome} — histogram, the peak resident set of that exec
|
|
12
12
|
* - cortex.sandbox.exec.cpu_seconds{outcome} — histogram, the CPU time of that exec
|
|
13
|
+
* - cortex.sandbox.lease.wait{backend} — histogram, from a lease request to its grant
|
|
14
|
+
* - cortex.sandbox.machine.start{backend} — histogram, from a spawn to the health of the machine
|
|
15
|
+
* - cortex.sandbox.machine.replaced{cause} — counter, one per machine of a lease that went away
|
|
13
16
|
* - cortex.artifact.reconcile.dropped{agent_id} — counter for missing manifest entries
|
|
14
17
|
* - cortex.artifact.reconcile.input_dropped{agent_id, reason} — counter for lineage input drops
|
|
15
18
|
*
|
|
@@ -31,6 +34,7 @@
|
|
|
31
34
|
* proxy: a meter taken at module load, before `initOtel` runs, would stay
|
|
32
35
|
* on the no-op provider for the life of the process.
|
|
33
36
|
*/
|
|
37
|
+
import type { WorkKind } from "../contracts/chat-parts.js";
|
|
34
38
|
import type { StepExecutionStatus } from "../state/schema.js";
|
|
35
39
|
/** Terminal status of a run, the same as the run ledger. */
|
|
36
40
|
export type RunOutcome = "completed" | "partial" | "failed" | "canceled";
|
|
@@ -71,6 +75,13 @@ export declare const SANDBOX_PEAK_MEMORY_BOUNDS_BYTES: readonly number[];
|
|
|
71
75
|
* granting.
|
|
72
76
|
*/
|
|
73
77
|
export declare const SANDBOX_CPU_SECONDS_BOUNDS: readonly number[];
|
|
78
|
+
/**
|
|
79
|
+
* Bucket bounds for a cold start and for a wait in the lease queue: 1, 5, 15, 30 s, then 1, 2, 5, 10, 30, 60 min.
|
|
80
|
+
* A machine on a ready node starts in seconds, and a new node or a full local budget takes minutes.
|
|
81
|
+
*/
|
|
82
|
+
export declare const SANDBOX_WAIT_BOUNDS_MS: readonly number[];
|
|
83
|
+
/** Why a machine of a lease went away. */
|
|
84
|
+
export type MachineReplacementCause = "idle" | "dead" | "oom" | "release";
|
|
74
85
|
/**
|
|
75
86
|
* Record one run that reached a terminal status. Call it inside the DBOS step
|
|
76
87
|
* that persists that status, or behind the ledger compare-and-set that accepted
|
|
@@ -122,6 +133,26 @@ export declare function recordSandboxExec(args: {
|
|
|
122
133
|
readonly peakMemoryBytes?: number | undefined;
|
|
123
134
|
readonly cpuMillis?: number | undefined;
|
|
124
135
|
}): void;
|
|
136
|
+
/** Record the wait of one lease from its request to its grant. Call it in the step that took the grant. */
|
|
137
|
+
export declare function recordLeaseWait(args: {
|
|
138
|
+
readonly backend: string;
|
|
139
|
+
readonly waitMs: number;
|
|
140
|
+
}): void;
|
|
141
|
+
/** Record the time from one spawn to the health of the machine. Call it in the step of the spawn. */
|
|
142
|
+
export declare function recordMachineStart(args: {
|
|
143
|
+
readonly backend: string;
|
|
144
|
+
readonly startMs: number;
|
|
145
|
+
}): void;
|
|
146
|
+
/** Record one machine of a lease that went away. Call it in the step that removed it. */
|
|
147
|
+
export declare function recordMachineReplaced(args: {
|
|
148
|
+
readonly cause: MachineReplacementCause;
|
|
149
|
+
}): void;
|
|
150
|
+
/** Count one lease workflow that runs in this process, until the returned function runs. */
|
|
151
|
+
export declare function trackLeaseWorkflow(): () => void;
|
|
152
|
+
/** Count one workflow body that waits in a pause in this process, until the returned function runs. */
|
|
153
|
+
export declare function trackWorkPaused(args: {
|
|
154
|
+
readonly workKind: WorkKind;
|
|
155
|
+
}): () => void;
|
|
125
156
|
/** Record one manifest entry dropped at reconcile. */
|
|
126
157
|
export declare function recordArtifactReconcileDropped(args: {
|
|
127
158
|
readonly agentId: string;
|
package/dist/lib/metrics.js
CHANGED
|
@@ -10,6 +10,9 @@
|
|
|
10
10
|
* - cortex.sandbox.exec.duration{outcome} — histogram, the runtime the sandbox reported for that exec
|
|
11
11
|
* - cortex.sandbox.exec.peak_memory_bytes{outcome} — histogram, the peak resident set of that exec
|
|
12
12
|
* - cortex.sandbox.exec.cpu_seconds{outcome} — histogram, the CPU time of that exec
|
|
13
|
+
* - cortex.sandbox.lease.wait{backend} — histogram, from a lease request to its grant
|
|
14
|
+
* - cortex.sandbox.machine.start{backend} — histogram, from a spawn to the health of the machine
|
|
15
|
+
* - cortex.sandbox.machine.replaced{cause} — counter, one per machine of a lease that went away
|
|
13
16
|
* - cortex.artifact.reconcile.dropped{agent_id} — counter for missing manifest entries
|
|
14
17
|
* - cortex.artifact.reconcile.input_dropped{agent_id, reason} — counter for lineage input drops
|
|
15
18
|
*
|
|
@@ -60,6 +63,11 @@ export const SANDBOX_PEAK_MEMORY_BOUNDS_BYTES = [
|
|
|
60
63
|
* granting.
|
|
61
64
|
*/
|
|
62
65
|
export const SANDBOX_CPU_SECONDS_BOUNDS = [0.5, 1, 5, 15, 60, 300, 900, 1_800, 3_600, 14_400];
|
|
66
|
+
/**
|
|
67
|
+
* Bucket bounds for a cold start and for a wait in the lease queue: 1, 5, 15, 30 s, then 1, 2, 5, 10, 30, 60 min.
|
|
68
|
+
* A machine on a ready node starts in seconds, and a new node or a full local budget takes minutes.
|
|
69
|
+
*/
|
|
70
|
+
export const SANDBOX_WAIT_BOUNDS_MS = [1_000, 5_000, 15_000, 30_000, 60_000, 120_000, 300_000, 600_000, 1_800_000, 3_600_000];
|
|
63
71
|
let instruments;
|
|
64
72
|
function getInstruments() {
|
|
65
73
|
if (instruments === undefined) {
|
|
@@ -103,6 +111,28 @@ function getInstruments() {
|
|
|
103
111
|
unit: "s",
|
|
104
112
|
advice: { explicitBucketBoundaries: [...SANDBOX_CPU_SECONDS_BOUNDS] },
|
|
105
113
|
}),
|
|
114
|
+
sandboxLeaseWait: meter.createHistogram("cortex.sandbox.lease.wait", {
|
|
115
|
+
description: "Wait of a sandbox lease from its request to its grant. Tagged by backend.",
|
|
116
|
+
unit: "ms",
|
|
117
|
+
advice: { explicitBucketBoundaries: [...SANDBOX_WAIT_BOUNDS_MS] },
|
|
118
|
+
}),
|
|
119
|
+
sandboxMachineStart: meter.createHistogram("cortex.sandbox.machine.start", {
|
|
120
|
+
description: "Time from the spawn of a sandbox machine to its health. Tagged by backend.",
|
|
121
|
+
unit: "ms",
|
|
122
|
+
advice: { explicitBucketBoundaries: [...SANDBOX_WAIT_BOUNDS_MS] },
|
|
123
|
+
}),
|
|
124
|
+
sandboxMachineReplaced: meter.createCounter("cortex.sandbox.machine.replaced", {
|
|
125
|
+
description: "Machines of a lease that went away. Tagged by cause.",
|
|
126
|
+
unit: "{machine}",
|
|
127
|
+
}),
|
|
128
|
+
sandboxLeaseWorkflows: meter.createUpDownCounter("cortex.sandbox.lease.workflows", {
|
|
129
|
+
description: "Lease workflows that run in this process. Each one waits in memory for the life of its lease.",
|
|
130
|
+
unit: "{workflow}",
|
|
131
|
+
}),
|
|
132
|
+
workPaused: meter.createUpDownCounter("cortex.work.paused", {
|
|
133
|
+
description: "Workflow bodies that wait in a pause in this process. Tagged by work_kind.",
|
|
134
|
+
unit: "{workflow}",
|
|
135
|
+
}),
|
|
106
136
|
artifactReconcileDropped: meter.createCounter("cortex.artifact.reconcile.dropped", {
|
|
107
137
|
description: "Manifest entries dropped because their on-disk file was missing at " +
|
|
108
138
|
"registration time (writes-then-deletes, renames). Tagged by agent_id.",
|
|
@@ -227,6 +257,40 @@ export function recordSandboxExec(args) {
|
|
|
227
257
|
if (args.cpuMillis !== undefined)
|
|
228
258
|
sandboxExecCpuSeconds.record(Math.max(0, args.cpuMillis) / 1000, attributes);
|
|
229
259
|
}
|
|
260
|
+
/** Record the wait of one lease from its request to its grant. Call it in the step that took the grant. */
|
|
261
|
+
export function recordLeaseWait(args) {
|
|
262
|
+
getInstruments().sandboxLeaseWait.record(Math.max(0, args.waitMs), { backend: args.backend });
|
|
263
|
+
}
|
|
264
|
+
/** Record the time from one spawn to the health of the machine. Call it in the step of the spawn. */
|
|
265
|
+
export function recordMachineStart(args) {
|
|
266
|
+
getInstruments().sandboxMachineStart.record(Math.max(0, args.startMs), { backend: args.backend });
|
|
267
|
+
}
|
|
268
|
+
/** Record one machine of a lease that went away. Call it in the step that removed it. */
|
|
269
|
+
export function recordMachineReplaced(args) {
|
|
270
|
+
getInstruments().sandboxMachineReplaced.add(1, { cause: args.cause });
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Count one value in this process until the returned function runs. The count is of this process only, thus a
|
|
274
|
+
* dashboard can put it beside the memory of the same process. A second call of the returned function does nothing.
|
|
275
|
+
*/
|
|
276
|
+
function track(counter, attributes) {
|
|
277
|
+
counter.add(1, attributes);
|
|
278
|
+
let open = true;
|
|
279
|
+
return () => {
|
|
280
|
+
if (!open)
|
|
281
|
+
return;
|
|
282
|
+
open = false;
|
|
283
|
+
counter.add(-1, attributes);
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
/** Count one lease workflow that runs in this process, until the returned function runs. */
|
|
287
|
+
export function trackLeaseWorkflow() {
|
|
288
|
+
return track(getInstruments().sandboxLeaseWorkflows);
|
|
289
|
+
}
|
|
290
|
+
/** Count one workflow body that waits in a pause in this process, until the returned function runs. */
|
|
291
|
+
export function trackWorkPaused(args) {
|
|
292
|
+
return track(getInstruments().workPaused, { work_kind: args.workKind });
|
|
293
|
+
}
|
|
230
294
|
/** Record one manifest entry dropped at reconcile. */
|
|
231
295
|
export function recordArtifactReconcileDropped(args) {
|
|
232
296
|
getInstruments().artifactReconcileDropped.add(1, { agent_id: args.agentId });
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The text of a PDF, for each page, on `unpdf`: the serverless build of PDF.js, with no native part and no worker
|
|
3
|
+
* file. Thus it runs under Node, under Bun, and in a Bun single-file binary.
|
|
4
|
+
*/
|
|
5
|
+
/** The text of one page. */
|
|
6
|
+
export type PdfPageText = {
|
|
7
|
+
readonly page: number;
|
|
8
|
+
readonly text: string;
|
|
9
|
+
};
|
|
10
|
+
/** The pages that a read gave, and the page count of the whole document. */
|
|
11
|
+
export type PdfText = {
|
|
12
|
+
readonly totalPages: number;
|
|
13
|
+
readonly pages: readonly PdfPageText[];
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Read the text of the pages `first` to `last` of a PDF, both inclusive and 1-based, clamped to the document. The
|
|
17
|
+
* whole document is never decoded: PDF.js parses a page when the read asks for it. A PDF that PDF.js cannot open
|
|
18
|
+
* throws, and the caller maps the throw.
|
|
19
|
+
*/
|
|
20
|
+
export declare function readPdfText(bytes: Uint8Array, range: {
|
|
21
|
+
readonly first: number;
|
|
22
|
+
readonly last: number;
|
|
23
|
+
}): Promise<PdfText>;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The text of a PDF, for each page, on `unpdf`: the serverless build of PDF.js, with no native part and no worker
|
|
3
|
+
* file. Thus it runs under Node, under Bun, and in a Bun single-file binary.
|
|
4
|
+
*/
|
|
5
|
+
import { getDocumentProxy } from "unpdf";
|
|
6
|
+
/** The text of the items of a page, with the line ends that PDF.js marks. A marked-content item carries no text. */
|
|
7
|
+
function pageText(items) {
|
|
8
|
+
return items
|
|
9
|
+
.map((item) => ("str" in item && typeof item.str === "string" ? `${item.str}${"hasEOL" in item && item.hasEOL === true ? "\n" : ""}` : ""))
|
|
10
|
+
.join("")
|
|
11
|
+
.trim();
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Read the text of the pages `first` to `last` of a PDF, both inclusive and 1-based, clamped to the document. The
|
|
15
|
+
* whole document is never decoded: PDF.js parses a page when the read asks for it. A PDF that PDF.js cannot open
|
|
16
|
+
* throws, and the caller maps the throw.
|
|
17
|
+
*/
|
|
18
|
+
export async function readPdfText(bytes, range) {
|
|
19
|
+
// The verbosity 0 keeps the warnings of PDF.js off the console of the host.
|
|
20
|
+
const pdf = await getDocumentProxy(new Uint8Array(bytes), { verbosity: 0 });
|
|
21
|
+
try {
|
|
22
|
+
const first = Math.max(1, range.first);
|
|
23
|
+
const last = Math.min(pdf.numPages, range.last);
|
|
24
|
+
const pages = [];
|
|
25
|
+
for (let page = first; page <= last; page++) {
|
|
26
|
+
const content = await (await pdf.getPage(page)).getTextContent();
|
|
27
|
+
pages.push({ page, text: pageText(content.items) });
|
|
28
|
+
}
|
|
29
|
+
return { totalPages: pdf.numPages, pages };
|
|
30
|
+
}
|
|
31
|
+
finally {
|
|
32
|
+
await pdf.cleanup();
|
|
33
|
+
}
|
|
34
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { load } from "cheerio";
|
|
2
|
-
import {
|
|
2
|
+
import { readPdfText } from "../pdf-text.js";
|
|
3
3
|
import { normalizeWhitespace } from "./normalize-whitespace.js";
|
|
4
4
|
/**
|
|
5
5
|
* Curated catalog of FDA guidance documents to index into the corpus.
|
|
@@ -77,10 +77,8 @@ export async function fetchAndExtractFdaDoc(stub) {
|
|
|
77
77
|
const ct = (res.headers.get("content-type") ?? "").toLowerCase();
|
|
78
78
|
let text;
|
|
79
79
|
if (ct.includes("pdf") || stub.doc_url.toLowerCase().endsWith(".pdf")) {
|
|
80
|
-
const
|
|
81
|
-
|
|
82
|
-
await parser.destroy();
|
|
83
|
-
text = result.text;
|
|
80
|
+
const result = await readPdfText(buf, { first: 1, last: Number.MAX_SAFE_INTEGER });
|
|
81
|
+
text = result.pages.map((page) => page.text).join("\n\n");
|
|
84
82
|
}
|
|
85
83
|
else {
|
|
86
84
|
const $ = load(buf.toString("utf-8"));
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* Codes look like S7A, E14, M3(R2), Q1A. The doc_id we emit is
|
|
12
12
|
* `ich-<code>` with non-word chars sanitized.
|
|
13
13
|
*/
|
|
14
|
-
import {
|
|
14
|
+
import { readPdfText } from "../pdf-text.js";
|
|
15
15
|
import { normalizeWhitespace } from "./normalize-whitespace.js";
|
|
16
16
|
/**
|
|
17
17
|
* Curated catalog of ICH guidelines to index into the corpus.
|
|
@@ -144,14 +144,12 @@ export async function fetchAndExtractIchDoc(stub) {
|
|
|
144
144
|
throw new Error(`fetch ${stub.doc_url}: ${res.status} ${res.statusText}`);
|
|
145
145
|
}
|
|
146
146
|
const buf = Buffer.from(await res.arrayBuffer());
|
|
147
|
-
const
|
|
148
|
-
const result = await parser.getText();
|
|
149
|
-
await parser.destroy();
|
|
147
|
+
const result = await readPdfText(buf, { first: 1, last: Number.MAX_SAFE_INTEGER });
|
|
150
148
|
return {
|
|
151
149
|
doc_id: stub.doc_id,
|
|
152
150
|
doc_title: stub.doc_title,
|
|
153
151
|
doc_url: stub.doc_url,
|
|
154
|
-
text: normalizeWhitespace(result.text),
|
|
152
|
+
text: normalizeWhitespace(result.pages.map((page) => page.text).join("\n\n")),
|
|
155
153
|
metadata: {
|
|
156
154
|
code: stub.code,
|
|
157
155
|
series: stub.series,
|
package/dist/loop/run-loop.d.ts
CHANGED
|
@@ -231,3 +231,8 @@ export type SalvagedResult = LoopResult & {
|
|
|
231
231
|
*/
|
|
232
232
|
export declare function runWithSalvage(agent: AgentDefinition, initial: readonly LoopMessage[], session: AgentSession, opts: LoopOptions, salvage: TerminalSalvage): Promise<SalvagedResult>;
|
|
233
233
|
export declare function finalText(messages: readonly LoopMessage[]): string;
|
|
234
|
+
/**
|
|
235
|
+
* The agent with the `run_script` tool over its script tools, after its own tools. The tool is a pure function of
|
|
236
|
+
* the script tools, thus each request of a conversation declares the same tool, and the prompt cache holds.
|
|
237
|
+
*/
|
|
238
|
+
export declare function withRunScript(agent: AgentDefinition, logger?: Logger): AgentDefinition;
|
package/dist/loop/run-loop.js
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
import { randomUUID } from "node:crypto";
|
|
17
17
|
import { DBOS } from "@dbos-inc/dbos-sdk";
|
|
18
18
|
import { durableCalls } from "@dbos-inc/vercel-ai";
|
|
19
|
-
import { streamText, wrapLanguageModel, } from "ai";
|
|
19
|
+
import { NoSuchToolError, streamText, wrapLanguageModel, } from "ai";
|
|
20
20
|
import { forSubAgent } from "../auth/types.js";
|
|
21
21
|
import { createNoopUsageRecorder } from "../billing/noop-usage-recorder.js";
|
|
22
22
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
@@ -25,11 +25,13 @@ import { errorNameOf } from "../lib/thrown.js";
|
|
|
25
25
|
import { contextRecordOf, markCompactionExchange, markInterruptedMessage, summaryMarkerMessage, syntheticUserMessage, withRequestInputTokens, } from "../memory/ai-sdk-message-storage.js";
|
|
26
26
|
import { conversationView, lastRequestInputTokens, withoutReasoning } from "../memory/conversation-view.js";
|
|
27
27
|
import { answerUnansweredToolCalls } from "../memory/tool-call-integrity.js";
|
|
28
|
+
import { aliasedToolName } from "./tool-name-repair.js";
|
|
28
29
|
import { fallbackModelOf } from "../providers/ai-sdk.js";
|
|
29
30
|
import { extractStatus, findProviderError } from "../providers/errors.js";
|
|
30
31
|
import { DEFAULT_PROMPT_CACHE, withPromptCacheBreakpoint, withSystemPromptBreakpoint } from "../providers/prompt-cache.js";
|
|
31
32
|
import { accountingMiddleware, rollupMiddleware } from "./accounting.js";
|
|
32
33
|
import { createCallPolicy } from "./call-policy.js";
|
|
34
|
+
import { createRunScriptTool } from "../tools/script/run-script.js";
|
|
33
35
|
import { COMPACTION_MAX_REQUESTS } from "./compaction.js";
|
|
34
36
|
import { addChatUsage, hasReportedUsage, recordAgentRun } from "./metrics.js";
|
|
35
37
|
import { adaptTools, deniedReason, foldCallRecords, isFatalToolError, modelViewOf, recordedInputOf, ToolCallOutputSchema, } from "./tool-adapter.js";
|
|
@@ -133,11 +135,28 @@ export function finalText(messages) {
|
|
|
133
135
|
.map((block) => block.text)
|
|
134
136
|
.join("");
|
|
135
137
|
}
|
|
138
|
+
/**
|
|
139
|
+
* The agent with the `run_script` tool over its script tools, after its own tools. The tool is a pure function of
|
|
140
|
+
* the script tools, thus each request of a conversation declares the same tool, and the prompt cache holds.
|
|
141
|
+
*/
|
|
142
|
+
export function withRunScript(agent, logger) {
|
|
143
|
+
const scriptTools = agent.scriptTools ?? [];
|
|
144
|
+
if (scriptTools.length === 0)
|
|
145
|
+
return agent;
|
|
146
|
+
const asking = scriptTools.find((tool) => tool.approval !== undefined || tool.startsWorkflow === true);
|
|
147
|
+
if (asking !== undefined)
|
|
148
|
+
throw new Error(`the script tool ${asking.id} of agent ${agent.id} asks an approval or starts a workflow`);
|
|
149
|
+
const ownStep = scriptTools.find((tool) => tool.ownStep === true);
|
|
150
|
+
if (ownStep !== undefined)
|
|
151
|
+
throw new Error(`the script tool ${ownStep.id} of agent ${agent.id} must run as its own durable step`);
|
|
152
|
+
return { ...agent, tools: [...agent.tools, createRunScriptTool(scriptTools, agent.scriptRuntime, { scratch: agent.scriptScratch, logger })] };
|
|
153
|
+
}
|
|
136
154
|
/**
|
|
137
155
|
* Open the loop over one conversation. `metricAgentId` is the id that usage
|
|
138
156
|
* and run metrics count under, which is not always `agent.id`.
|
|
139
157
|
*/
|
|
140
|
-
function openConversation(
|
|
158
|
+
function openConversation(declared, initial, session, opts, metricAgentId) {
|
|
159
|
+
const agent = withRunScript(declared, opts.logger);
|
|
141
160
|
const { provider, emit } = opts;
|
|
142
161
|
if (agent.tools.length > 0 && !provider.capabilities.toolCalling) {
|
|
143
162
|
throw new Error(`Provider/model cannot run tool-required agent "${agent.id}"`);
|
|
@@ -516,6 +535,15 @@ async function runSeries(agent, opts, calls, log, target, spec) {
|
|
|
516
535
|
maxRetries: 0,
|
|
517
536
|
abortSignal: AbortSignal.any([opts.signal, fatalController.signal]),
|
|
518
537
|
telemetry: bound.telemetry,
|
|
538
|
+
repairToolCall: ({ toolCall, tools: declared, error }) => {
|
|
539
|
+
if (!NoSuchToolError.isInstance(error))
|
|
540
|
+
return Promise.resolve(null);
|
|
541
|
+
const tool = aliasedToolName(toolCall.toolName, new Set(Object.keys(declared)));
|
|
542
|
+
if (tool === undefined)
|
|
543
|
+
return Promise.resolve(null);
|
|
544
|
+
log.warn("a tool call named a proxy alias of a declared tool", { alias: toolCall.toolName, tool });
|
|
545
|
+
return Promise.resolve({ ...toolCall, toolName: tool });
|
|
546
|
+
},
|
|
519
547
|
toolApproval: async ({ toolCall }) => {
|
|
520
548
|
const refusal = calls.refusalOf({ toolName: toolCall.toolName, input: toolCall.input }, spec.mask);
|
|
521
549
|
if (refusal !== undefined) {
|
package/dist/loop/sub-agent.js
CHANGED
|
@@ -23,7 +23,7 @@ import { DATA_PROFILE_RUN_LITERAL } from "../contracts/data-profile.js";
|
|
|
23
23
|
import { createSubAgentActivityTracker } from "../contracts/subagent-activity.js";
|
|
24
24
|
import { CHAT_STREAM_KEY, createTurnStreamWriter, SUB_AGENT_STREAM_PART } from "./chat-stream.js";
|
|
25
25
|
import { addChatUsage } from "./metrics.js";
|
|
26
|
-
import { credentialGuard, currentRootTurn, turnCredentials, withRootTurn } from "./turn-credentials.js";
|
|
26
|
+
import { credentialGuard, currentRootTurn, liveAuth, turnCredentials, withRootTurn } from "./turn-credentials.js";
|
|
27
27
|
/** The work of a child, from its input. A data profile runs under one constant frame and starts no child, thus it gives none. */
|
|
28
28
|
function workOf(input) {
|
|
29
29
|
const { analysisId, threadId } = input.scope;
|
|
@@ -59,11 +59,10 @@ export function registerSubAgent(name, body, pauses) {
|
|
|
59
59
|
scope: input.scope,
|
|
60
60
|
provenance: input.provenance,
|
|
61
61
|
runFrame: input.runFrame,
|
|
62
|
-
// The credential comes at each read, thus a call after a resume of the
|
|
63
|
-
// resume put. A child with no credential makes no model call, thus no
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
},
|
|
62
|
+
// The credential comes at each read, also through a copy of the session, thus a call after a resume of the
|
|
63
|
+
// turn uses the credential that the resume put. A child with no credential makes no model call, thus no
|
|
64
|
+
// adapter reads the empty value.
|
|
65
|
+
auth: liveAuth(authOf),
|
|
67
66
|
};
|
|
68
67
|
// The input holds the provenance of the caller. The body of the sub-agent extends it.
|
|
69
68
|
const caller = { agentId: input.provenance.agentId, callPath: [...input.provenance.callPath] };
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The repair of a tool name that a proxy alias left behind.
|
|
3
|
+
*
|
|
4
|
+
* The Claude cloak of CLIProxyAPI renames each client tool to `mcp__<server>__<word>_<tool>`, and it maps the name
|
|
5
|
+
* back on the response. The model sometimes drops the `mcp__<server>__` part. Then the proxy maps nothing back, and
|
|
6
|
+
* the loop gets `<word>_<tool>`, for example `art_read_file` (router-for-me/CLIProxyAPI#6848, which upstream closed as
|
|
7
|
+
* not planned). A name of one lowercase word, an underscore, and a declared tool id is such an alias. The repair
|
|
8
|
+
* applies only to a name that no tool declares, thus a declared tool always keeps its own name.
|
|
9
|
+
*/
|
|
10
|
+
/** The declared tool that an alias names, or `undefined` for a name that is not an alias of one. */
|
|
11
|
+
export declare function aliasedToolName(name: string, declared: ReadonlySet<string>): string | undefined;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The repair of a tool name that a proxy alias left behind.
|
|
3
|
+
*
|
|
4
|
+
* The Claude cloak of CLIProxyAPI renames each client tool to `mcp__<server>__<word>_<tool>`, and it maps the name
|
|
5
|
+
* back on the response. The model sometimes drops the `mcp__<server>__` part. Then the proxy maps nothing back, and
|
|
6
|
+
* the loop gets `<word>_<tool>`, for example `art_read_file` (router-for-me/CLIProxyAPI#6848, which upstream closed as
|
|
7
|
+
* not planned). A name of one lowercase word, an underscore, and a declared tool id is such an alias. The repair
|
|
8
|
+
* applies only to a name that no tool declares, thus a declared tool always keeps its own name.
|
|
9
|
+
*/
|
|
10
|
+
/** The declared tool that an alias names, or `undefined` for a name that is not an alias of one. */
|
|
11
|
+
export function aliasedToolName(name, declared) {
|
|
12
|
+
if (declared.has(name))
|
|
13
|
+
return undefined;
|
|
14
|
+
const split = name.indexOf("_");
|
|
15
|
+
if (split <= 0)
|
|
16
|
+
return undefined;
|
|
17
|
+
const word = name.slice(0, split);
|
|
18
|
+
const tool = name.slice(split + 1);
|
|
19
|
+
for (const char of word)
|
|
20
|
+
if (char < "a" || char > "z")
|
|
21
|
+
return undefined;
|
|
22
|
+
return declared.has(tool) ? tool : undefined;
|
|
23
|
+
}
|
|
@@ -17,6 +17,11 @@ export declare const turnCredentials: {
|
|
|
17
17
|
/** The function of the host for a turn that this process did not open. */
|
|
18
18
|
setHostCredential(credentialFor: ((turnId: string) => AuthContext | undefined) | undefined): void;
|
|
19
19
|
};
|
|
20
|
+
/**
|
|
21
|
+
* The `auth` of a session whose credential can come later: each read goes to `current`, and no credential reads as an
|
|
22
|
+
* empty object. A copy of the session keeps this object, thus each copy sees the credential that a resume puts.
|
|
23
|
+
*/
|
|
24
|
+
export declare function liveAuth(current: () => AuthContext | undefined): AuthContext;
|
|
20
25
|
/** Run the body of a turn, or of one of its sub-agents, with the id of its root turn in its context. */
|
|
21
26
|
export declare function withRootTurn<T>(turnId: string, body: () => Promise<T>): Promise<T>;
|
|
22
27
|
/** The id of the root chat turn of the body that runs, or `undefined` outside a chat turn. */
|