@alisio/core 0.1.0-alpha.21 → 0.1.0-alpha.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analysis/capabilities.d.ts +48 -0
- package/dist/analysis/capabilities.js +90 -0
- package/dist/analysis/data/csv.d.ts +46 -0
- package/dist/analysis/data/csv.js +240 -0
- package/dist/analysis/data/datasets.d.ts +132 -0
- package/dist/analysis/data/datasets.js +441 -0
- package/dist/analysis/data/describe.d.ts +27 -0
- package/dist/analysis/data/describe.js +103 -0
- package/dist/analysis/data/engine-client.d.ts +58 -0
- package/dist/analysis/data/engine-client.js +304 -0
- package/dist/analysis/data/engine-entry.d.ts +1 -0
- package/dist/analysis/data/engine-entry.js +59 -0
- package/dist/analysis/data/engine-source.d.ts +4 -0
- package/dist/analysis/data/engine-source.js +6 -0
- package/dist/analysis/data/engine-types.d.ts +86 -0
- package/dist/analysis/data/engine-types.js +7 -0
- package/dist/analysis/data/infer.d.ts +42 -0
- package/dist/analysis/data/infer.js +118 -0
- package/dist/analysis/data/ingest-worker.d.ts +17 -0
- package/dist/analysis/data/ingest-worker.js +365 -0
- package/dist/analysis/data/json.d.ts +19 -0
- package/dist/analysis/data/json.js +44 -0
- package/dist/analysis/data/query-worker.d.ts +12 -0
- package/dist/analysis/data/query-worker.js +75 -0
- package/dist/analysis/data/rows.d.ts +39 -0
- package/dist/analysis/data/rows.js +120 -0
- package/dist/analysis/data/sql-guard.d.ts +20 -0
- package/dist/analysis/data/sql-guard.js +129 -0
- package/dist/analysis/data/stats.d.ts +18 -0
- package/dist/analysis/data/stats.js +77 -0
- package/dist/analysis/install-hints.d.ts +33 -0
- package/dist/analysis/install-hints.js +144 -0
- package/dist/analysis/janitor.d.ts +60 -0
- package/dist/analysis/janitor.js +264 -0
- package/dist/analysis/jobs.d.ts +94 -0
- package/dist/analysis/jobs.js +165 -0
- package/dist/analysis/oci.d.ts +129 -0
- package/dist/analysis/oci.js +330 -0
- package/dist/analysis/python/sources.d.ts +2 -0
- package/dist/analysis/python/sources.js +10 -0
- package/dist/analysis/requirements.d.ts +32 -0
- package/dist/analysis/requirements.js +73 -0
- package/dist/analysis/rerun.d.ts +40 -0
- package/dist/analysis/rerun.js +136 -0
- package/dist/analysis/runtime-manager.d.ts +146 -0
- package/dist/analysis/runtime-manager.js +472 -0
- package/dist/analysis/status.d.ts +21 -0
- package/dist/analysis/status.js +52 -0
- package/dist/application.d.ts +41 -0
- package/dist/application.js +195 -0
- package/dist/artifacts/kinds.d.ts +35 -0
- package/dist/artifacts/kinds.js +177 -0
- package/dist/artifacts/publisher.d.ts +7 -0
- package/dist/artifacts/publisher.js +29 -0
- package/dist/artifacts/store.d.ts +119 -0
- package/dist/artifacts/store.js +581 -0
- package/dist/commands/builtins.js +13 -0
- package/dist/config.d.ts +102 -0
- package/dist/config.js +167 -6
- package/dist/core/contracts.d.ts +69 -1
- package/dist/core/runner.d.ts +34 -4
- package/dist/core/runner.js +388 -130
- package/dist/index.d.ts +23 -4
- package/dist/index.js +22 -3
- package/dist/mcp/rich.js +4 -0
- package/dist/plugins/host.js +14 -6
- package/dist/runtime/blobs.d.ts +22 -0
- package/dist/runtime/blobs.js +84 -1
- package/dist/runtime/ids.d.ts +8 -0
- package/dist/runtime/ids.js +32 -0
- package/dist/runtime/paths.d.ts +5 -0
- package/dist/runtime/paths.js +6 -0
- package/dist/runtime/process.d.ts +12 -0
- package/dist/runtime/process.js +64 -4
- package/dist/runtime/sqlite.d.ts +2 -0
- package/dist/runtime/sqlite.js +18 -0
- package/dist/runtime/store.d.ts +5 -0
- package/dist/runtime/store.js +75 -0
- package/dist/runtime/zip.d.ts +24 -0
- package/dist/runtime/zip.js +126 -0
- package/dist/sessions/children.js +4 -0
- package/dist/tools/analysis.d.ts +37 -0
- package/dist/tools/analysis.js +572 -0
- package/dist/tools/artifacts.d.ts +14 -0
- package/dist/tools/artifacts.js +143 -0
- package/dist/tools/data.d.ts +7 -0
- package/dist/tools/data.js +160 -0
- package/dist/tools/execute.js +6 -2
- package/package.json +2 -2
package/dist/application.js
CHANGED
|
@@ -2,6 +2,15 @@ import { realpath } from "node:fs/promises";
|
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
3
|
import { dirname, join, resolve } from "node:path";
|
|
4
4
|
import { AgentDefinitionService } from "./agents/service.js";
|
|
5
|
+
import { CapabilityGrants } from "./analysis/capabilities.js";
|
|
6
|
+
import { DatasetService } from "./analysis/data/datasets.js";
|
|
7
|
+
import { AnalysisJanitor } from "./analysis/janitor.js";
|
|
8
|
+
import { AnalysisJobs } from "./analysis/jobs.js";
|
|
9
|
+
import { OciRuntime } from "./analysis/oci.js";
|
|
10
|
+
import { AnalysisRerun } from "./analysis/rerun.js";
|
|
11
|
+
import { AnalysisRuntimeManager } from "./analysis/runtime-manager.js";
|
|
12
|
+
import { createArtifactPublisher } from "./artifacts/publisher.js";
|
|
13
|
+
import { ArtifactStore } from "./artifacts/store.js";
|
|
5
14
|
import { configFile, configHome, loadConfigWithProvenance, overridesSavedProviderProfile, setConfigValue, setGlobalMcpAllow, setMcpServerEnabled, setProjectPluginEnabled, setProjectSkillEnabled, stateHome, } from "./config.js";
|
|
6
15
|
import { completeText } from "./core/compaction.js";
|
|
7
16
|
import { ToolRegistry } from "./core/registry.js";
|
|
@@ -268,6 +277,105 @@ export async function createApplication(options = {}) {
|
|
|
268
277
|
config: config.websearch,
|
|
269
278
|
resolveExtension: () => plugins.extensions.resolve("websearch"),
|
|
270
279
|
}, pathAccess);
|
|
280
|
+
// Python analysis and artifacts live next to the session database (like attachment blobs).
|
|
281
|
+
const stateRoot = options.db && options.db !== ":memory:" ? dirname(options.db) : stateHome();
|
|
282
|
+
const artifacts = new ArtifactStore({
|
|
283
|
+
root: stateRoot,
|
|
284
|
+
db: store.db,
|
|
285
|
+
limits: {
|
|
286
|
+
maxFiles: config.analysis.limits.maxFiles,
|
|
287
|
+
maxFileBytes: config.analysis.limits.maxFileBytes,
|
|
288
|
+
maxOutputBytes: config.analysis.limits.maxOutputBytes,
|
|
289
|
+
},
|
|
290
|
+
});
|
|
291
|
+
const capabilityGrants = new CapabilityGrants({
|
|
292
|
+
db: store.db,
|
|
293
|
+
rootOf: (id) => store.rootOf(id),
|
|
294
|
+
});
|
|
295
|
+
const analysisRuntime = options.analysisRuntime ??
|
|
296
|
+
new AnalysisRuntimeManager({
|
|
297
|
+
stateDir: stateRoot,
|
|
298
|
+
...(options.python ? { python: options.python } : {}),
|
|
299
|
+
});
|
|
300
|
+
const analysisJobs = new AnalysisJobs({ root: stateRoot, db: store.db });
|
|
301
|
+
// Container mode (`analysis.runtime: "oci"`): read from the live config on every call.
|
|
302
|
+
let ociRuntime;
|
|
303
|
+
const oci = {
|
|
304
|
+
mode: () => config.analysis.runtime,
|
|
305
|
+
runtime: () => {
|
|
306
|
+
const key = JSON.stringify(config.analysis.oci);
|
|
307
|
+
if (ociRuntime?.key !== key)
|
|
308
|
+
ociRuntime = {
|
|
309
|
+
key,
|
|
310
|
+
runtime: new OciRuntime({ ...config.analysis.oci }, options.analysisOci),
|
|
311
|
+
};
|
|
312
|
+
return ociRuntime.runtime;
|
|
313
|
+
},
|
|
314
|
+
};
|
|
315
|
+
// Tabular datasets (one SQLite file each); XLSX needs the discovered Python.
|
|
316
|
+
const datasets = new DatasetService({
|
|
317
|
+
root: stateRoot,
|
|
318
|
+
db: store.db,
|
|
319
|
+
limits: config.analysis.data,
|
|
320
|
+
python: analysisRuntime,
|
|
321
|
+
});
|
|
322
|
+
const analysisRerun = new AnalysisRerun({ jobs: analysisJobs, store: artifacts, datasets });
|
|
323
|
+
// Retention: a sweep at most every 24 h, launched in the background after the start.
|
|
324
|
+
const janitor = new AnalysisJanitor({
|
|
325
|
+
root: stateRoot,
|
|
326
|
+
db: store.db,
|
|
327
|
+
artifacts,
|
|
328
|
+
datasets,
|
|
329
|
+
blobs,
|
|
330
|
+
retention: () => config.analysis.retention,
|
|
331
|
+
...(options.analysisSweepDelayMs !== undefined
|
|
332
|
+
? { initialDelayMs: options.analysisSweepDelayMs }
|
|
333
|
+
: {}),
|
|
334
|
+
});
|
|
335
|
+
// R4: nothing is registered with analysis disabled or under --read-only. Startup only looks
|
|
336
|
+
// for candidate interpreters with `stat`; no process is launched until the first call.
|
|
337
|
+
const analysisEnabled = config.analysis.enabled && !options.readOnly;
|
|
338
|
+
if (analysisEnabled) {
|
|
339
|
+
const [candidates, active] = await Promise.all([
|
|
340
|
+
analysisRuntime.candidates(),
|
|
341
|
+
analysisRuntime.activeRuntime(),
|
|
342
|
+
]);
|
|
343
|
+
const { registerAnalysisTools } = await import("./tools/analysis.js");
|
|
344
|
+
registerAnalysisTools(registry, {
|
|
345
|
+
store: artifacts,
|
|
346
|
+
jobs: analysisJobs,
|
|
347
|
+
runtime: analysisRuntime,
|
|
348
|
+
datasets,
|
|
349
|
+
oci,
|
|
350
|
+
rerun: analysisRerun,
|
|
351
|
+
rootOf: (id) => store.rootOf(id),
|
|
352
|
+
// Read live: a changed `analysis.limits.*` setting applies to the next call.
|
|
353
|
+
limits: {
|
|
354
|
+
get timeoutMs() {
|
|
355
|
+
return config.analysis.limits.timeoutMs;
|
|
356
|
+
},
|
|
357
|
+
get maxLogBytes() {
|
|
358
|
+
return config.analysis.limits.maxLogBytes;
|
|
359
|
+
},
|
|
360
|
+
},
|
|
361
|
+
startup: { candidates: candidates.length > 0, extras: active?.extras ?? [] },
|
|
362
|
+
});
|
|
363
|
+
const { registerArtifactTools } = await import("./tools/artifacts.js");
|
|
364
|
+
registerArtifactTools(registry, { store: artifacts, rootOf: (id) => store.rootOf(id) });
|
|
365
|
+
}
|
|
366
|
+
// Data tools only read (effect `read`): they stay available under --read-only.
|
|
367
|
+
if (config.analysis.enabled) {
|
|
368
|
+
const { registerDataTools } = await import("./tools/data.js");
|
|
369
|
+
registerDataTools(registry, { datasets, rootOf: (id) => store.rootOf(id) });
|
|
370
|
+
}
|
|
371
|
+
const sessionWorkspace = (sessionId) => {
|
|
372
|
+
try {
|
|
373
|
+
return store.get(sessionId).workspace;
|
|
374
|
+
}
|
|
375
|
+
catch {
|
|
376
|
+
return workspace;
|
|
377
|
+
}
|
|
378
|
+
};
|
|
271
379
|
const { registerPluginInstallTool } = await import("./plugins/install.js");
|
|
272
380
|
registerPluginInstallTool(registry, { readOnly: !!options.readOnly });
|
|
273
381
|
if ((options.allowMcp || mcpAllow) && !options.readOnly)
|
|
@@ -508,6 +616,7 @@ export async function createApplication(options = {}) {
|
|
|
508
616
|
const runtimePolicy = {
|
|
509
617
|
write: !!options.allowWrite && !options.readOnly,
|
|
510
618
|
process: !!options.allowProcess && !options.readOnly,
|
|
619
|
+
...(options.allowAnalysis && !options.readOnly ? { analysis: true } : {}),
|
|
511
620
|
external: !options.readOnly &&
|
|
512
621
|
(!!options.allowExternal ||
|
|
513
622
|
!!options.allowMcp ||
|
|
@@ -516,6 +625,19 @@ export async function createApplication(options = {}) {
|
|
|
516
625
|
plugins.externalCount > 0 ||
|
|
517
626
|
registry.list().some((t) => t.name.startsWith("p_"))),
|
|
518
627
|
};
|
|
628
|
+
/** Model and provider of a session, for the provenance of its artifacts. */
|
|
629
|
+
const provenanceOf = (sessionId) => {
|
|
630
|
+
try {
|
|
631
|
+
const session = store.get(sessionId);
|
|
632
|
+
return {
|
|
633
|
+
...(session.model ? { model: session.model } : {}),
|
|
634
|
+
...(session.provider ? { provider: session.provider } : {}),
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
catch {
|
|
638
|
+
return {};
|
|
639
|
+
}
|
|
640
|
+
};
|
|
519
641
|
const runner = new AgentRunner({
|
|
520
642
|
provider,
|
|
521
643
|
providerFor,
|
|
@@ -530,6 +652,33 @@ export async function createApplication(options = {}) {
|
|
|
530
652
|
contextWindow,
|
|
531
653
|
...(options.approve && !options.readOnly ? { approve: options.approve } : {}),
|
|
532
654
|
pathAccess,
|
|
655
|
+
...(analysisEnabled
|
|
656
|
+
? {
|
|
657
|
+
capabilities: {
|
|
658
|
+
granted: (capability, sessionId) => capabilityGrants.granted(capability, sessionId),
|
|
659
|
+
record: (input) => {
|
|
660
|
+
capabilityGrants.record({ ...input, workspace: sessionWorkspace(input.sessionId) });
|
|
661
|
+
},
|
|
662
|
+
source: options.approvalSource ?? "tui",
|
|
663
|
+
get runtime() {
|
|
664
|
+
return config.analysis.runtime;
|
|
665
|
+
},
|
|
666
|
+
// A rerun's approval shows the saved script (the input carries only its reference).
|
|
667
|
+
preview: async (input, sessionId) => typeof input.rerunOf === "string"
|
|
668
|
+
? analysisRerun.script(input.rerunOf, store.rootOf(sessionId))
|
|
669
|
+
: undefined,
|
|
670
|
+
},
|
|
671
|
+
}
|
|
672
|
+
: {}),
|
|
673
|
+
artifacts: (call, announce) => createArtifactPublisher(artifacts, {
|
|
674
|
+
sessionId: call.sessionId,
|
|
675
|
+
rootSessionId: store.rootOf(call.sessionId),
|
|
676
|
+
workspace: sessionWorkspace(call.sessionId),
|
|
677
|
+
runId: call.runId,
|
|
678
|
+
callId: call.callId,
|
|
679
|
+
// Public provenance of what produced the artifact (no paths, no keys).
|
|
680
|
+
provenance: provenanceOf(call.sessionId),
|
|
681
|
+
}, announce),
|
|
533
682
|
...(config.websearch.provider === "native"
|
|
534
683
|
? { nativeTools: [{ type: config.websearch.nativeToolType }] }
|
|
535
684
|
: {}),
|
|
@@ -612,6 +761,9 @@ export async function createApplication(options = {}) {
|
|
|
612
761
|
}
|
|
613
762
|
}
|
|
614
763
|
}
|
|
764
|
+
// Retention runs in the background (unreferenced timer); never for in-memory test databases.
|
|
765
|
+
if (analysisEnabled && options.db !== ":memory:")
|
|
766
|
+
janitor.start();
|
|
615
767
|
return {
|
|
616
768
|
workspace,
|
|
617
769
|
pathAccess,
|
|
@@ -621,6 +773,24 @@ export async function createApplication(options = {}) {
|
|
|
621
773
|
store,
|
|
622
774
|
/** Uploaded attachment bytes; resolve a `BlobRef` with `blobs.attachment(ref)` before a run. */
|
|
623
775
|
blobs,
|
|
776
|
+
/** Published artifacts of every session (folders under `<state>/artifacts`). */
|
|
777
|
+
artifacts,
|
|
778
|
+
/** Persisted capability grants (`analysis.run`) and their audit rows. */
|
|
779
|
+
capabilityGrants,
|
|
780
|
+
/** Python discovery and job folders of `python_run`; `enabled` is false under --read-only. */
|
|
781
|
+
analysis: {
|
|
782
|
+
enabled: analysisEnabled,
|
|
783
|
+
runtime: analysisRuntime,
|
|
784
|
+
jobs: analysisJobs,
|
|
785
|
+
/** Container runtime of the current configuration (status, pull); mode `managed` ignores it. */
|
|
786
|
+
oci,
|
|
787
|
+
rerun: analysisRerun,
|
|
788
|
+
janitor,
|
|
789
|
+
},
|
|
790
|
+
/** Keys a project layer set that only the user layer decides (ignored on load). */
|
|
791
|
+
configDiagnostics: configProvenance.ignored ?? [],
|
|
792
|
+
/** Tabular datasets of every session (ingestion, read-only queries, pages). */
|
|
793
|
+
datasets,
|
|
624
794
|
registry,
|
|
625
795
|
context,
|
|
626
796
|
skills,
|
|
@@ -842,6 +1012,29 @@ export async function createApplication(options = {}) {
|
|
|
842
1012
|
? { active: config.agents.active }
|
|
843
1013
|
: { ...config.agents, effort: String(value) };
|
|
844
1014
|
break;
|
|
1015
|
+
case "analysis.enabled":
|
|
1016
|
+
// Tools are registered when the application starts: the change applies to the next
|
|
1017
|
+
// application (a restart; the web recycles the workspace application).
|
|
1018
|
+
config.analysis = { ...config.analysis, enabled: value === true };
|
|
1019
|
+
break;
|
|
1020
|
+
case "analysis.limits.timeoutMs":
|
|
1021
|
+
// Read live by python_run on its next call.
|
|
1022
|
+
config.analysis = {
|
|
1023
|
+
...config.analysis,
|
|
1024
|
+
limits: { ...config.analysis.limits, timeoutMs: Number(value) },
|
|
1025
|
+
};
|
|
1026
|
+
break;
|
|
1027
|
+
case "analysis.retention.jobsDays":
|
|
1028
|
+
case "analysis.retention.intermediateDays":
|
|
1029
|
+
case "analysis.retention.artifactsDays": {
|
|
1030
|
+
// Read by the next sweep.
|
|
1031
|
+
const leaf = key.slice("analysis.retention.".length);
|
|
1032
|
+
config.analysis = {
|
|
1033
|
+
...config.analysis,
|
|
1034
|
+
retention: { ...config.analysis.retention, [leaf]: Number(value) },
|
|
1035
|
+
};
|
|
1036
|
+
break;
|
|
1037
|
+
}
|
|
845
1038
|
case "websearch.provider":
|
|
846
1039
|
// Mutated in place (same object identity) so the tool chain's per-call
|
|
847
1040
|
// `websearchCtx.config` sees the new provider on the very next search call.
|
|
@@ -998,6 +1191,8 @@ export async function createApplication(options = {}) {
|
|
|
998
1191
|
// component cannot stall /exit; teardown errors are swallowed for callers.
|
|
999
1192
|
const capped = (work) => Promise.race([work.catch(() => { }), delay(TEARDOWN_STAGE_TIMEOUT_MS)]);
|
|
1000
1193
|
try {
|
|
1194
|
+
janitor.stop();
|
|
1195
|
+
datasets?.close();
|
|
1001
1196
|
await capped(Promise.allSettled([herdr.close(), mcp.close()]));
|
|
1002
1197
|
}
|
|
1003
1198
|
finally {
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Artifact type registry: the kind, the served MIME type and whether a rich client may preview a
|
|
3
|
+
* published file. The type comes from the extension AND the leading bytes; when they disagree the
|
|
4
|
+
* most restrictive type wins (`file`, download only). The served `Content-Type` always comes from
|
|
5
|
+
* here, never from the script that wrote the file.
|
|
6
|
+
*/
|
|
7
|
+
import type { ArtifactKind } from "@alisio/sdk";
|
|
8
|
+
export interface ArtifactType {
|
|
9
|
+
kind: ArtifactKind;
|
|
10
|
+
mimeType: string;
|
|
11
|
+
}
|
|
12
|
+
export declare const DOWNLOAD_ONLY: ArtifactType;
|
|
13
|
+
/** Lower-case extension without the dot (`""` when none). */
|
|
14
|
+
export declare const extensionOf: (fileName: string) => string;
|
|
15
|
+
/** Text: no NUL byte in the sampled head and the sample decodes as UTF-8. */
|
|
16
|
+
export declare function looksLikeText(head: Uint8Array): boolean;
|
|
17
|
+
/**
|
|
18
|
+
* The type of a file from its name and its first bytes (at least 4 KiB is a good sample). An
|
|
19
|
+
* unknown extension, or bytes that contradict the extension, yield `file` (download only).
|
|
20
|
+
* An empty file keeps the type of its extension.
|
|
21
|
+
*/
|
|
22
|
+
export declare function classifyArtifact(fileName: string, head: Uint8Array): ArtifactType;
|
|
23
|
+
/** Whether a rich client may preview a file of this name, type and size. */
|
|
24
|
+
export declare function isPreviewable(fileName: string, type: ArtifactType, bytes: number): boolean;
|
|
25
|
+
/** English labels of each kind (the web localizes its own; text surfaces use these). */
|
|
26
|
+
export declare const KIND_LABELS: Record<ArtifactKind, string>;
|
|
27
|
+
/** The type the sandboxed viewer serves for one file (`application/octet-stream` when unknown). */
|
|
28
|
+
export declare const viewerContentType: (fileName: string) => string;
|
|
29
|
+
/**
|
|
30
|
+
* The registry type of one file inside an artifact (for the authenticated `files/*` route):
|
|
31
|
+
* the extension and the leading bytes decide, exactly like a published file.
|
|
32
|
+
*/
|
|
33
|
+
export declare function fileType(fileName: string, head: Uint8Array): ArtifactType & {
|
|
34
|
+
inline: boolean;
|
|
35
|
+
};
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
const MB = 1024 * 1024;
|
|
2
|
+
const TEXT_PREVIEW = 2 * MB;
|
|
3
|
+
/** Tables are read through paged queries, not loaded whole: the limit is the ingestion limit. */
|
|
4
|
+
const TABLE_PREVIEW = 200 * MB;
|
|
5
|
+
const UTF8 = "; charset=utf-8";
|
|
6
|
+
const OOXML = "application/vnd.openxmlformats-officedocument";
|
|
7
|
+
const text = (kind, mimeType, previewMax = TEXT_PREVIEW) => ({
|
|
8
|
+
kind,
|
|
9
|
+
mimeType,
|
|
10
|
+
magic: "text",
|
|
11
|
+
previewMax,
|
|
12
|
+
});
|
|
13
|
+
const RULES = {
|
|
14
|
+
html: text("dashboard", `text/html${UTF8}`, 20 * MB),
|
|
15
|
+
htm: text("dashboard", `text/html${UTF8}`, 20 * MB),
|
|
16
|
+
md: text("document", `text/markdown${UTF8}`),
|
|
17
|
+
markdown: text("document", `text/markdown${UTF8}`),
|
|
18
|
+
pdf: { kind: "document", mimeType: "application/pdf", magic: "pdf", previewMax: 50 * MB },
|
|
19
|
+
docx: { kind: "document", mimeType: `${OOXML}.wordprocessingml.document`, magic: "zip" },
|
|
20
|
+
pptx: { kind: "document", mimeType: `${OOXML}.presentationml.presentation`, magic: "zip" },
|
|
21
|
+
odt: { kind: "document", mimeType: "application/vnd.oasis.opendocument.text", magic: "zip" },
|
|
22
|
+
// Tables open in SpreadsheetView (phase 3): CSV/TSV may be windows-1252 or UTF-16, which the
|
|
23
|
+
// ingestion decodes, so their bytes only have to look like delimited text, not like UTF-8.
|
|
24
|
+
csv: {
|
|
25
|
+
kind: "spreadsheet",
|
|
26
|
+
mimeType: `text/csv${UTF8}`,
|
|
27
|
+
magic: "delimited",
|
|
28
|
+
previewMax: TABLE_PREVIEW,
|
|
29
|
+
},
|
|
30
|
+
tsv: {
|
|
31
|
+
kind: "spreadsheet",
|
|
32
|
+
mimeType: `text/tab-separated-values${UTF8}`,
|
|
33
|
+
magic: "delimited",
|
|
34
|
+
previewMax: TABLE_PREVIEW,
|
|
35
|
+
},
|
|
36
|
+
xlsx: {
|
|
37
|
+
kind: "spreadsheet",
|
|
38
|
+
mimeType: `${OOXML}.spreadsheetml.sheet`,
|
|
39
|
+
magic: "zip",
|
|
40
|
+
previewMax: TABLE_PREVIEW,
|
|
41
|
+
},
|
|
42
|
+
png: { kind: "image", mimeType: "image/png", magic: "png", previewMax: 20 * MB },
|
|
43
|
+
jpg: { kind: "image", mimeType: "image/jpeg", magic: "jpeg", previewMax: 20 * MB },
|
|
44
|
+
jpeg: { kind: "image", mimeType: "image/jpeg", magic: "jpeg", previewMax: 20 * MB },
|
|
45
|
+
gif: { kind: "image", mimeType: "image/gif", magic: "gif", previewMax: 20 * MB },
|
|
46
|
+
webp: { kind: "image", mimeType: "image/webp", magic: "webp", previewMax: 20 * MB },
|
|
47
|
+
svg: text("image", "image/svg+xml", 20 * MB),
|
|
48
|
+
json: text("data", "application/json"),
|
|
49
|
+
geojson: text("data", "application/json"),
|
|
50
|
+
zip: { kind: "archive", mimeType: "application/zip", magic: "zip" },
|
|
51
|
+
};
|
|
52
|
+
for (const ext of ["txt", "log", "sql", "yaml", "yml", "xml", "py", "js", "ts", "r"])
|
|
53
|
+
RULES[ext] = text("code", `text/plain${UTF8}`);
|
|
54
|
+
export const DOWNLOAD_ONLY = { kind: "file", mimeType: "application/octet-stream" };
|
|
55
|
+
/** Lower-case extension without the dot (`""` when none). */
|
|
56
|
+
export const extensionOf = (fileName) => {
|
|
57
|
+
const base = fileName.split(/[\\/]/).pop() ?? "";
|
|
58
|
+
const dot = base.lastIndexOf(".");
|
|
59
|
+
return dot > 0 ? base.slice(dot + 1).toLowerCase() : "";
|
|
60
|
+
};
|
|
61
|
+
const startsWith = (bytes, signature, at = 0) => bytes.length >= at + signature.length && signature.every((b, i) => bytes[at + i] === b);
|
|
62
|
+
/** Text: no NUL byte in the sampled head and the sample decodes as UTF-8. */
|
|
63
|
+
export function looksLikeText(head) {
|
|
64
|
+
if (head.includes(0))
|
|
65
|
+
return false;
|
|
66
|
+
try {
|
|
67
|
+
// A cut multi-byte character at the end of the sample is not a decoding failure.
|
|
68
|
+
new TextDecoder("utf-8", { fatal: true }).decode(head.subarray(0, Math.max(0, head.length - 3)));
|
|
69
|
+
return true;
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
function matches(magic, head) {
|
|
76
|
+
switch (magic) {
|
|
77
|
+
case "text":
|
|
78
|
+
return looksLikeText(head);
|
|
79
|
+
case "delimited":
|
|
80
|
+
// Text in any of the encodings the ingestion reads: UTF-8, windows-1252 (no NUL bytes) or
|
|
81
|
+
// UTF-16 with a BOM.
|
|
82
|
+
return (looksLikeText(head) ||
|
|
83
|
+
startsWith(head, [0xff, 0xfe]) ||
|
|
84
|
+
startsWith(head, [0xfe, 0xff]) ||
|
|
85
|
+
!head.includes(0));
|
|
86
|
+
case "pdf":
|
|
87
|
+
return startsWith(head, [0x25, 0x50, 0x44, 0x46, 0x2d]);
|
|
88
|
+
case "png":
|
|
89
|
+
return startsWith(head, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
|
90
|
+
case "jpeg":
|
|
91
|
+
return startsWith(head, [0xff, 0xd8, 0xff]);
|
|
92
|
+
case "gif":
|
|
93
|
+
return startsWith(head, [0x47, 0x49, 0x46, 0x38]);
|
|
94
|
+
case "webp":
|
|
95
|
+
return (startsWith(head, [0x52, 0x49, 0x46, 0x46]) && startsWith(head, [0x57, 0x45, 0x42, 0x50], 8));
|
|
96
|
+
case "zip":
|
|
97
|
+
return (startsWith(head, [0x50, 0x4b, 0x03, 0x04]) || startsWith(head, [0x50, 0x4b, 0x05, 0x06]));
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* The type of a file from its name and its first bytes (at least 4 KiB is a good sample). An
|
|
102
|
+
* unknown extension, or bytes that contradict the extension, yield `file` (download only).
|
|
103
|
+
* An empty file keeps the type of its extension.
|
|
104
|
+
*/
|
|
105
|
+
export function classifyArtifact(fileName, head) {
|
|
106
|
+
const rule = RULES[extensionOf(fileName)];
|
|
107
|
+
if (!rule)
|
|
108
|
+
return DOWNLOAD_ONLY;
|
|
109
|
+
if (head.length > 0 && !matches(rule.magic, head))
|
|
110
|
+
return DOWNLOAD_ONLY;
|
|
111
|
+
return { kind: rule.kind, mimeType: rule.mimeType };
|
|
112
|
+
}
|
|
113
|
+
/** Whether a rich client may preview a file of this name, type and size. */
|
|
114
|
+
export function isPreviewable(fileName, type, bytes) {
|
|
115
|
+
if (type.kind === "file" || type.kind === "archive")
|
|
116
|
+
return false;
|
|
117
|
+
const rule = RULES[extensionOf(fileName)];
|
|
118
|
+
return !!rule?.previewMax && rule.mimeType === type.mimeType && bytes <= rule.previewMax;
|
|
119
|
+
}
|
|
120
|
+
/** English labels of each kind (the web localizes its own; text surfaces use these). */
|
|
121
|
+
export const KIND_LABELS = {
|
|
122
|
+
dashboard: "Dashboard",
|
|
123
|
+
document: "Document",
|
|
124
|
+
spreadsheet: "Spreadsheet",
|
|
125
|
+
image: "Image",
|
|
126
|
+
data: "Data",
|
|
127
|
+
code: "Text",
|
|
128
|
+
archive: "Archive",
|
|
129
|
+
file: "File",
|
|
130
|
+
};
|
|
131
|
+
/**
|
|
132
|
+
* Content types of the files of an artifact served by the isolated viewer (spec §14.2): a
|
|
133
|
+
* multi-file dashboard needs its scripts, styles and fonts with their real types (under
|
|
134
|
+
* `nosniff` a script served as text never runs). Only that sandboxed route uses this table;
|
|
135
|
+
* every other route serves the registry type above.
|
|
136
|
+
*/
|
|
137
|
+
const VIEWER_TYPES = {
|
|
138
|
+
html: `text/html${UTF8}`,
|
|
139
|
+
htm: `text/html${UTF8}`,
|
|
140
|
+
css: `text/css${UTF8}`,
|
|
141
|
+
js: `text/javascript${UTF8}`,
|
|
142
|
+
mjs: `text/javascript${UTF8}`,
|
|
143
|
+
json: `application/json${UTF8}`,
|
|
144
|
+
geojson: `application/json${UTF8}`,
|
|
145
|
+
csv: `text/csv${UTF8}`,
|
|
146
|
+
tsv: `text/tab-separated-values${UTF8}`,
|
|
147
|
+
txt: `text/plain${UTF8}`,
|
|
148
|
+
md: `text/markdown${UTF8}`,
|
|
149
|
+
svg: "image/svg+xml",
|
|
150
|
+
png: "image/png",
|
|
151
|
+
jpg: "image/jpeg",
|
|
152
|
+
jpeg: "image/jpeg",
|
|
153
|
+
gif: "image/gif",
|
|
154
|
+
webp: "image/webp",
|
|
155
|
+
ico: "image/x-icon",
|
|
156
|
+
woff: "font/woff",
|
|
157
|
+
woff2: "font/woff2",
|
|
158
|
+
ttf: "font/ttf",
|
|
159
|
+
otf: "font/otf",
|
|
160
|
+
pdf: "application/pdf",
|
|
161
|
+
mp4: "video/mp4",
|
|
162
|
+
webm: "video/webm",
|
|
163
|
+
mp3: "audio/mpeg",
|
|
164
|
+
wav: "audio/wav",
|
|
165
|
+
};
|
|
166
|
+
/** The type the sandboxed viewer serves for one file (`application/octet-stream` when unknown). */
|
|
167
|
+
export const viewerContentType = (fileName) => VIEWER_TYPES[extensionOf(fileName)] ?? "application/octet-stream";
|
|
168
|
+
/**
|
|
169
|
+
* The registry type of one file inside an artifact (for the authenticated `files/*` route):
|
|
170
|
+
* the extension and the leading bytes decide, exactly like a published file.
|
|
171
|
+
*/
|
|
172
|
+
export function fileType(fileName, head) {
|
|
173
|
+
const type = classifyArtifact(fileName, head);
|
|
174
|
+
// HTML never renders on the app origin; download-only kinds never render inline.
|
|
175
|
+
const inline = type.kind !== "file" && type.kind !== "archive" && type.kind !== "dashboard";
|
|
176
|
+
return { ...type, inline };
|
|
177
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `ToolContext.artifacts` of one tool call: binds the artifact store to the calling session
|
|
3
|
+
* (and its root), run and call, and announces every publication (`artifact_published`).
|
|
4
|
+
*/
|
|
5
|
+
import type { CoreArtifactPublisher, PublishedArtifactInfo } from "../core/contracts.ts";
|
|
6
|
+
import type { ArtifactOwner, ArtifactStore } from "./store.ts";
|
|
7
|
+
export declare function createArtifactPublisher(store: ArtifactStore, owner: ArtifactOwner, announce: (published: PublishedArtifactInfo) => void): CoreArtifactPublisher;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export function createArtifactPublisher(store, owner, announce) {
|
|
2
|
+
const done = (published, executionId) => {
|
|
3
|
+
const info = { ...published, ...(executionId ? { executionId } : {}) };
|
|
4
|
+
announce(info);
|
|
5
|
+
return info;
|
|
6
|
+
};
|
|
7
|
+
return {
|
|
8
|
+
async publish(input) {
|
|
9
|
+
return done(await store.publish(input, owner)).artifact;
|
|
10
|
+
},
|
|
11
|
+
async publishText(input) {
|
|
12
|
+
return done(await store.publishText(input, owner)).artifact;
|
|
13
|
+
},
|
|
14
|
+
async publishTextDetailed(input) {
|
|
15
|
+
return done(await store.publishText(input, owner));
|
|
16
|
+
},
|
|
17
|
+
async publishOutputs(staging, options) {
|
|
18
|
+
const published = await store.publishOutputs(staging, {
|
|
19
|
+
...owner,
|
|
20
|
+
executionId: options.executionId,
|
|
21
|
+
...(options.partial ? { partial: true } : {}),
|
|
22
|
+
...(options.provenance
|
|
23
|
+
? { provenance: { ...owner.provenance, ...options.provenance } }
|
|
24
|
+
: {}),
|
|
25
|
+
}, options.title ? { title: options.title } : {});
|
|
26
|
+
return published.map((p) => done(p, options.executionId));
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import type { ArtifactKind, ArtifactPublishInput, ArtifactRef, SqlDatabase } from "@alisio/sdk";
|
|
2
|
+
export interface ArtifactLimits {
|
|
3
|
+
maxFiles: number;
|
|
4
|
+
maxFileBytes: number;
|
|
5
|
+
/** Total bytes of one publication (one execution). */
|
|
6
|
+
maxOutputBytes: number;
|
|
7
|
+
}
|
|
8
|
+
export declare const DEFAULT_ARTIFACT_LIMITS: ArtifactLimits;
|
|
9
|
+
/** Who publishes: the session (and its root), the call that produced the files, provenance. */
|
|
10
|
+
export interface ArtifactOwner {
|
|
11
|
+
sessionId: string;
|
|
12
|
+
rootSessionId: string;
|
|
13
|
+
workspace: string;
|
|
14
|
+
runId?: string;
|
|
15
|
+
callId?: string;
|
|
16
|
+
executionId?: string;
|
|
17
|
+
partial?: boolean;
|
|
18
|
+
/** Extra public provenance (runtime, script hash, inputs, model, provider). No paths. */
|
|
19
|
+
provenance?: Record<string, unknown>;
|
|
20
|
+
}
|
|
21
|
+
export interface PublishedArtifact {
|
|
22
|
+
artifact: ArtifactRef;
|
|
23
|
+
/** Absolute path of the file, or of the entry of a multi-file artifact. */
|
|
24
|
+
path: string;
|
|
25
|
+
/** Non-fatal notes (e.g. a dashboard reference to a missing file). */
|
|
26
|
+
warnings: string[];
|
|
27
|
+
}
|
|
28
|
+
export interface ArtifactFile {
|
|
29
|
+
path: string;
|
|
30
|
+
bytes: number;
|
|
31
|
+
sha256: string;
|
|
32
|
+
}
|
|
33
|
+
export interface ArtifactRecord extends ArtifactRef {
|
|
34
|
+
entry?: string;
|
|
35
|
+
executionId?: string;
|
|
36
|
+
runId?: string;
|
|
37
|
+
callId?: string;
|
|
38
|
+
provenance: Record<string, unknown>;
|
|
39
|
+
/** Folder relative to the store root. */
|
|
40
|
+
relDir: string;
|
|
41
|
+
}
|
|
42
|
+
/** A rejected publication: the message is user-readable and names the offending path. */
|
|
43
|
+
export declare class ArtifactRejected extends Error {
|
|
44
|
+
}
|
|
45
|
+
/** `title` → `[a-z0-9-]`, at most 48 characters (`artifact` when nothing remains). */
|
|
46
|
+
export declare function slugify(title: string): string;
|
|
47
|
+
/** The first bytes of a file (the sample `classifyArtifact` needs). */
|
|
48
|
+
export declare function readHead(path: string): Promise<Uint8Array>;
|
|
49
|
+
/** The public part of a record (what the SDK `ArtifactRef` carries). */
|
|
50
|
+
export declare const toRef: (record: ArtifactRecord) => ArtifactRef;
|
|
51
|
+
export declare class ArtifactStore {
|
|
52
|
+
private options;
|
|
53
|
+
readonly limits: ArtifactLimits;
|
|
54
|
+
private readonly now;
|
|
55
|
+
constructor(options: {
|
|
56
|
+
/** State root (`stateHome()` or the folder of `--db`); artifacts live under `artifacts/`. */
|
|
57
|
+
root: string;
|
|
58
|
+
db: SqlDatabase;
|
|
59
|
+
limits?: Partial<ArtifactLimits>;
|
|
60
|
+
now?: () => number;
|
|
61
|
+
});
|
|
62
|
+
get root(): string;
|
|
63
|
+
/** Session folder of an owner (artifacts of child sessions live with their root). */
|
|
64
|
+
private sessionDir;
|
|
65
|
+
/**
|
|
66
|
+
* Publishes the outputs a script left in `staging`: the entries of `staging/outputs.json`, or
|
|
67
|
+
* else every top-level file (and every top-level folder; with `index.html` a multi-file
|
|
68
|
+
* dashboard, otherwise one ZIP archive). Everything or nothing is published.
|
|
69
|
+
*/
|
|
70
|
+
publishOutputs(staging: string, owner: ArtifactOwner, defaults?: {
|
|
71
|
+
title?: string;
|
|
72
|
+
}): Promise<PublishedArtifact[]>;
|
|
73
|
+
/** Publishes one caller-owned file or directory. */
|
|
74
|
+
publish(input: ArtifactPublishInput, owner: ArtifactOwner): Promise<PublishedArtifact>;
|
|
75
|
+
/** Publishes a single text file the caller holds in memory. */
|
|
76
|
+
publishText(input: {
|
|
77
|
+
fileName: string;
|
|
78
|
+
title?: string;
|
|
79
|
+
text: string;
|
|
80
|
+
}, owner: ArtifactOwner): Promise<PublishedArtifact>;
|
|
81
|
+
private scan;
|
|
82
|
+
private publishAll;
|
|
83
|
+
private build;
|
|
84
|
+
/** Artifacts of a root session, newest first. */
|
|
85
|
+
list(rootSessionId: string, options?: {
|
|
86
|
+
kind?: ArtifactKind;
|
|
87
|
+
limit?: number;
|
|
88
|
+
before?: number;
|
|
89
|
+
/** Also `deleted` and `expired` rows (the folder is gone, the row remains). */
|
|
90
|
+
includeDeleted?: boolean;
|
|
91
|
+
/** Also `expired` rows (retention), but not `deleted` ones. */
|
|
92
|
+
includeExpired?: boolean;
|
|
93
|
+
}): ArtifactRecord[];
|
|
94
|
+
get(id: string): ArtifactRecord | undefined;
|
|
95
|
+
/** Absolute folder holding `manifest.json` and `files/`. */
|
|
96
|
+
folder(record: ArtifactRecord): string;
|
|
97
|
+
/** Absolute path of the file (or the entry of a multi-file artifact). */
|
|
98
|
+
localPath(record: ArtifactRecord): string;
|
|
99
|
+
/** Files of an artifact from its manifest, with absolute paths (inside `files/`). */
|
|
100
|
+
files(record: ArtifactRecord): Promise<Array<ArtifactFile & {
|
|
101
|
+
abs: string;
|
|
102
|
+
}>>;
|
|
103
|
+
/**
|
|
104
|
+
* One file of a ready artifact by its manifest path (`files/` only, never `manifest.json`):
|
|
105
|
+
* the path must be listed in the manifest and its real path must stay inside `files/`.
|
|
106
|
+
*/
|
|
107
|
+
resolveFile(record: ArtifactRecord, path: string): Promise<(ArtifactFile & {
|
|
108
|
+
abs: string;
|
|
109
|
+
}) | undefined>;
|
|
110
|
+
/**
|
|
111
|
+
* Retention: marks a `ready` artifact `expired` and removes its folder; the row stays so cards
|
|
112
|
+
* and listings keep showing it as `Expired`. False when it is not `ready`.
|
|
113
|
+
*/
|
|
114
|
+
markExpired(id: string): Promise<boolean>;
|
|
115
|
+
/** Ready artifacts created before `cutoff` (epoch ms), oldest first (for the retention sweep). */
|
|
116
|
+
readyBefore(cutoff: number): ArtifactRecord[];
|
|
117
|
+
/** Marks an artifact deleted and removes its folder (the row stays for references). */
|
|
118
|
+
delete(id: string): Promise<boolean>;
|
|
119
|
+
}
|