@arcforge/platform 2.0.166
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/package.json +69 -0
- package/src/bin/supervisor.ts +102 -0
- package/src/bin/test-api.ts +30 -0
- package/src/bin/test-preload.ts +308 -0
- package/src/bin/update-helper.ts +32 -0
- package/src/build/bench/CLAUDE.md +106 -0
- package/src/build/bench/SPEC.md +209 -0
- package/src/build/bench/agents.ts +105 -0
- package/src/build/bench/axes.ts +84 -0
- package/src/build/bench/bench.ts +96 -0
- package/src/build/bench/discover.ts +147 -0
- package/src/build/bench/index.ts +7 -0
- package/src/build/bench/log.ts +47 -0
- package/src/build/bench/matrix.ts +23 -0
- package/src/build/bench/preload.ts +440 -0
- package/src/build/bench/prepare.ts +61 -0
- package/src/build/bench/projection.ts +112 -0
- package/src/build/bench/resolve.ts +54 -0
- package/src/build/bench/runner.ts +309 -0
- package/src/build/bench/schema/extract.ts +148 -0
- package/src/build/bench/schema/index.ts +2 -0
- package/src/build/bench/schema/schema.ts +95 -0
- package/src/build/bench/schema/worker.ts +30 -0
- package/src/build/bench/typegen.ts +119 -0
- package/src/build/bench/workspace/files.ts +139 -0
- package/src/build/bench/workspace/handle.ts +133 -0
- package/src/build/bench/workspace/index.ts +11 -0
- package/src/build/bench/workspace/workspace.ts +14 -0
- package/src/build/blueprint/blueprint.ts +398 -0
- package/src/build/blueprint/cognet/abi.ts +207 -0
- package/src/build/blueprint/cognet/bundle-worker.ts +93 -0
- package/src/build/blueprint/cognet/bundle.ts +664 -0
- package/src/build/blueprint/cognet/cognet.ts +249 -0
- package/src/build/blueprint/cognet/index.ts +10 -0
- package/src/build/blueprint/cognet/manifest.ts +38 -0
- package/src/build/blueprint/cognet/resolve.ts +113 -0
- package/src/build/blueprint/collisions.ts +51 -0
- package/src/build/blueprint/index.ts +29 -0
- package/src/build/blueprint/modules/entries.ts +117 -0
- package/src/build/blueprint/modules/index.ts +300 -0
- package/src/build/blueprint/modules/meta.ts +174 -0
- package/src/build/blueprint/scan/boot.ts +34 -0
- package/src/build/blueprint/scan/cognetImports.ts +56 -0
- package/src/build/blueprint/scan/components.ts +45 -0
- package/src/build/blueprint/scan/config.ts +234 -0
- package/src/build/blueprint/scan/configImports.ts +232 -0
- package/src/build/blueprint/scan/declare-worker.ts +61 -0
- package/src/build/blueprint/scan/declare.ts +393 -0
- package/src/build/blueprint/scan/h3-globals.ts +137 -0
- package/src/build/blueprint/scan/knowledge.ts +101 -0
- package/src/build/blueprint/scan/middleware.ts +110 -0
- package/src/build/blueprint/scan/moduleImports.ts +79 -0
- package/src/build/blueprint/scan/plugins.ts +93 -0
- package/src/build/blueprint/scan/promptPackages.ts +45 -0
- package/src/build/blueprint/scan/prompts.ts +164 -0
- package/src/build/blueprint/scan/published.ts +131 -0
- package/src/build/blueprint/scan/resolve.ts +398 -0
- package/src/build/blueprint/scan/routes.ts +109 -0
- package/src/build/blueprint/scan/scripts.ts +119 -0
- package/src/build/blueprint/scan/tool-bundle-worker.ts +69 -0
- package/src/build/blueprint/scan/tools.ts +610 -0
- package/src/build/blueprint/types.ts +28 -0
- package/src/build/extensions/config.ts +261 -0
- package/src/build/extensions/edit.ts +684 -0
- package/src/build/extensions/extensions.ts +731 -0
- package/src/build/extensions/index.ts +10 -0
- package/src/build/extensions/load.ts +350 -0
- package/src/build/extensions/registrations.ts +189 -0
- package/src/build/extensions/store.ts +200 -0
- package/src/build/frame/frame.ts +145 -0
- package/src/build/frame/index.ts +5 -0
- package/src/build/frame/migrate.ts +246 -0
- package/src/build/project/bundle/agent.ts +382 -0
- package/src/build/project/bundle/artifacts.ts +88 -0
- package/src/build/project/bundle/assets.ts +385 -0
- package/src/build/project/bundle/bundle.ts +160 -0
- package/src/build/project/bundle/imports.ts +102 -0
- package/src/build/project/bundle/index.ts +16 -0
- package/src/build/project/bundle/module.ts +130 -0
- package/src/build/project/bundle/source.ts +136 -0
- package/src/build/project/bundle/stage.ts +97 -0
- package/src/build/project/bundle/types.ts +111 -0
- package/src/build/project/cognet.ts +156 -0
- package/src/build/project/create/agent.ts +52 -0
- package/src/build/project/create/bench.ts +81 -0
- package/src/build/project/create/cognet.ts +102 -0
- package/src/build/project/create/extension.ts +63 -0
- package/src/build/project/create/module.ts +43 -0
- package/src/build/project/create/profile.ts +102 -0
- package/src/build/project/create/prompt.ts +68 -0
- package/src/build/project/create/templates.ts +88 -0
- package/src/build/project/deploy.ts +96 -0
- package/src/build/project/index.ts +40 -0
- package/src/build/project/installer.ts +616 -0
- package/src/build/project/kinds.ts +374 -0
- package/src/build/project/links.ts +33 -0
- package/src/build/project/manifest/bunfig.ts +139 -0
- package/src/build/project/manifest/config.ts +246 -0
- package/src/build/project/manifest/env.ts +184 -0
- package/src/build/project/manifest/index.ts +9 -0
- package/src/build/project/manifest/manifest.ts +49 -0
- package/src/build/project/manifest/model.ts +124 -0
- package/src/build/project/manifest/package.ts +386 -0
- package/src/build/project/manifest/text.ts +19 -0
- package/src/build/project/models/fetch.ts +123 -0
- package/src/build/project/models/index.ts +8 -0
- package/src/build/project/models/models.ts +111 -0
- package/src/build/project/models/specifier.ts +106 -0
- package/src/build/project/models/store.ts +202 -0
- package/src/build/project/modules/hash.ts +31 -0
- package/src/build/project/modules/index.ts +5 -0
- package/src/build/project/modules/lock.ts +78 -0
- package/src/build/project/modules/modules.ts +155 -0
- package/src/build/project/prepare.ts +709 -0
- package/src/build/project/project.ts +253 -0
- package/src/build/project/projects.ts +164 -0
- package/src/build/project/propagation.ts +120 -0
- package/src/build/project/publish/verify.ts +312 -0
- package/src/build/project/publish.ts +155 -0
- package/src/build/project/reconcile.ts +93 -0
- package/src/build/project/specifier.ts +42 -0
- package/src/build/project/tree.ts +615 -0
- package/src/build/project/treecache.ts +356 -0
- package/src/build/project/typegen/axon-dts.ts +362 -0
- package/src/build/project/typegen/cognet-dts.ts +91 -0
- package/src/build/project/typegen/components.ts +45 -0
- package/src/build/project/typegen/env.ts +59 -0
- package/src/build/project/typegen/generators.ts +108 -0
- package/src/build/project/typegen/hooks.ts +71 -0
- package/src/build/project/typegen/index.ts +137 -0
- package/src/build/project/typegen/prompt-dts.ts +40 -0
- package/src/build/project/typegen/prompts.ts +36 -0
- package/src/build/project/typegen/registry-dts.ts +109 -0
- package/src/build/project/typegen/scripts.ts +37 -0
- package/src/build/project/typegen/tools.ts +31 -0
- package/src/build/project/typegen/tsconfig.ts +243 -0
- package/src/build/project/typegen/tui-dts.ts +120 -0
- package/src/build/project/typegen/write.ts +40 -0
- package/src/build/project/verify.ts +268 -0
- package/src/build/project/watcher.ts +266 -0
- package/src/build/runtime/agent.ts +550 -0
- package/src/build/runtime/escalations.ts +159 -0
- package/src/build/runtime/index.ts +15 -0
- package/src/build/runtime/instances.ts +604 -0
- package/src/build/runtime/profile.ts +91 -0
- package/src/build/runtime/progress.ts +143 -0
- package/src/build/runtime/requests.ts +167 -0
- package/src/build/runtime/resolve.ts +159 -0
- package/src/build/runtime/runtime.ts +233 -0
- package/src/build/runtime/sessions/index.ts +5 -0
- package/src/build/runtime/sessions/record.ts +145 -0
- package/src/build/runtime/sessions/sessions.ts +240 -0
- package/src/build/runtime/zeno.ts +120 -0
- package/src/index.ts +11 -0
- package/src/link/agent-main.ts +395 -0
- package/src/link/index.ts +70 -0
- package/src/link/services.ts +120 -0
- package/src/platform.ts +293 -0
- package/src/procs/index.ts +15 -0
- package/src/procs/tree.ts +177 -0
- package/src/services/cloud.ts +419 -0
- package/src/services/control/client.ts +126 -0
- package/src/services/control/dispatch.ts +156 -0
- package/src/services/control/index.ts +17 -0
- package/src/services/control/protocol.ts +179 -0
- package/src/services/control/server.ts +241 -0
- package/src/services/deployments.ts +81 -0
- package/src/services/files.ts +329 -0
- package/src/services/index.ts +5 -0
- package/src/services/mic/capture.ts +248 -0
- package/src/services/mic/config.ts +109 -0
- package/src/services/mic/fft.ts +93 -0
- package/src/services/mic/index.ts +139 -0
- package/src/services/mic/visualizer.ts +46 -0
- package/src/services/ollama/catalog.ts +122 -0
- package/src/services/ollama/http.ts +124 -0
- package/src/services/ollama/index.ts +13 -0
- package/src/services/ollama/models.ts +153 -0
- package/src/services/ollama/ollama.ts +89 -0
- package/src/services/ollama/registry.ts +112 -0
- package/src/services/ollama/types.ts +72 -0
- package/src/services/prompts.ts +76 -0
- package/src/services/registry.ts +220 -0
- package/src/services/resources/hardware.ts +125 -0
- package/src/services/resources/index.ts +6 -0
- package/src/services/resources/reservations.ts +162 -0
- package/src/services/resources/resources.ts +156 -0
- package/src/services/running/index.ts +1 -0
- package/src/services/running/running.ts +200 -0
- package/src/services/store/disk.ts +97 -0
- package/src/services/store/index.ts +15 -0
- package/src/services/store/store.ts +729 -0
- package/src/services/store/types.ts +303 -0
- package/src/services/test/child.ts +150 -0
- package/src/services/test/events.ts +102 -0
- package/src/services/test/files.ts +26 -0
- package/src/services/test/frames.ts +39 -0
- package/src/services/test/index.ts +7 -0
- package/src/services/test/runner.ts +122 -0
- package/src/services/test/types.ts +26 -0
- package/src/services/update/contract.ts +77 -0
- package/src/services/update/index.ts +12 -0
- package/src/services/update/installer.ts +139 -0
- package/src/services/update/updates.ts +137 -0
- package/src/utils/fs.ts +148 -0
- package/src/utils/tsast.ts +102 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { err } from "@arcforge/err"
|
|
2
|
+
import type { ModelRef } from "@arcforge/types"
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* A model specifier, parsed.
|
|
6
|
+
*
|
|
7
|
+
* `hf:owner/repo/path/to/file.onnx` is the short form and covers almost
|
|
8
|
+
* everything. Parsing is total and eager: a malformed specifier throws HERE,
|
|
9
|
+
* at prepare, naming the cognet field — never at fetch time with a confusing
|
|
10
|
+
* 404, and never at first inference with a missing file.
|
|
11
|
+
*/
|
|
12
|
+
export type ParsedModel = {
|
|
13
|
+
/** The local name the cognet addresses this weight by. */
|
|
14
|
+
key: string
|
|
15
|
+
host: "hf"
|
|
16
|
+
/** `owner/repo` */
|
|
17
|
+
repo: string
|
|
18
|
+
/** Path within the repo. A repo is a directory; big ones hold many weights. */
|
|
19
|
+
file: string
|
|
20
|
+
/** Git revision. `main` unless pinned. */
|
|
21
|
+
rev: string
|
|
22
|
+
/** Expected content hash, when the author pinned one. */
|
|
23
|
+
sha256?: string
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** The one place a model's fetch URL is constructed. */
|
|
27
|
+
export function downloadUrl(model: ParsedModel): string {
|
|
28
|
+
return `https://huggingface.co/${model.repo}/resolve/${model.rev}/${model.file}`
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** The filename a cached model is stored under — the last path segment. */
|
|
32
|
+
export function basenameOf(model: ParsedModel): string {
|
|
33
|
+
return model.file.split("/").pop()!
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Parse one declared model.
|
|
38
|
+
*
|
|
39
|
+
* The string form splits on the FIRST TWO path segments being the repo and
|
|
40
|
+
* everything after being the file, because HF repos nest arbitrarily:
|
|
41
|
+
* `hf:ggerganov/whisper.cpp/ggml-base.en.bin` is repo `ggerganov/whisper.cpp`
|
|
42
|
+
* and file `ggml-base.en.bin`, while
|
|
43
|
+
* `hf:onnx-community/silero-vad/onnx/model.onnx` keeps `onnx/model.onnx`
|
|
44
|
+
* whole.
|
|
45
|
+
*/
|
|
46
|
+
export function parseModel(key: string, ref: ModelRef): ParsedModel {
|
|
47
|
+
if (typeof ref !== "string") {
|
|
48
|
+
if (!ref.hf || !ref.file) {
|
|
49
|
+
throw err("MODEL_SPECIFIER_INVALID", {
|
|
50
|
+
detail: `models.${key}: the object form needs both "hf" (owner/repo) and "file"`,
|
|
51
|
+
context: { key },
|
|
52
|
+
})
|
|
53
|
+
}
|
|
54
|
+
return {
|
|
55
|
+
key,
|
|
56
|
+
host: "hf",
|
|
57
|
+
repo: ref.hf,
|
|
58
|
+
file: ref.file,
|
|
59
|
+
rev: ref.rev ?? "main",
|
|
60
|
+
...(ref.sha256 ? { sha256: ref.sha256 } : {}),
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const colon = ref.indexOf(":")
|
|
65
|
+
if (colon === -1) {
|
|
66
|
+
throw err("MODEL_SPECIFIER_INVALID", {
|
|
67
|
+
detail: `models.${key}: "${ref}" has no scheme — expected "hf:owner/repo/file"`,
|
|
68
|
+
context: { key, specifier: ref },
|
|
69
|
+
})
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const scheme = ref.slice(0, colon)
|
|
73
|
+
if (scheme !== "hf") {
|
|
74
|
+
throw err("MODEL_SPECIFIER_INVALID", {
|
|
75
|
+
detail: `models.${key}: unknown scheme "${scheme}" — only "hf:" is supported today`,
|
|
76
|
+
context: { key, specifier: ref, scheme },
|
|
77
|
+
})
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const segments = ref.slice(colon + 1).split("/").filter(Boolean)
|
|
81
|
+
if (segments.length < 3) {
|
|
82
|
+
// Two segments is a bare repo, which is ambiguous: a repo is a
|
|
83
|
+
// directory, and `ggerganov/whisper.cpp` alone names twenty
|
|
84
|
+
// quantisations of the same model. The file is not optional.
|
|
85
|
+
throw err("MODEL_SPECIFIER_INVALID", {
|
|
86
|
+
detail:
|
|
87
|
+
`models.${key}: "${ref}" names a repo but no file — a repo holds many weights, `
|
|
88
|
+
+ `so say which (e.g. "hf:owner/repo/model.onnx")`,
|
|
89
|
+
context: { key, specifier: ref },
|
|
90
|
+
})
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return {
|
|
94
|
+
key,
|
|
95
|
+
host: "hf",
|
|
96
|
+
repo: `${segments[0]}/${segments[1]}`,
|
|
97
|
+
file: segments.slice(2).join("/"),
|
|
98
|
+
rev: "main",
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Parse a whole `models:` declaration. Throws on the first malformed entry. */
|
|
103
|
+
export function parseModels(models: Record<string, ModelRef> | undefined): ParsedModel[] {
|
|
104
|
+
if (!models) return []
|
|
105
|
+
return Object.entries(models).map(([key, ref]) => parseModel(key, ref))
|
|
106
|
+
}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { createHash } from "node:crypto"
|
|
2
|
+
import { mkdir, mkdtemp, rename, rm, stat } from "node:fs/promises"
|
|
3
|
+
import { homedir, tmpdir } from "node:os"
|
|
4
|
+
import { join } from "node:path"
|
|
5
|
+
import { err } from "@arcforge/err"
|
|
6
|
+
import { fsx } from "../../../utils/fs"
|
|
7
|
+
import { basenameOf, type ParsedModel } from "./specifier"
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* ModelStore — the machine's cache of model weights.
|
|
11
|
+
*
|
|
12
|
+
* ```
|
|
13
|
+
* ~/.axon/models/<sha256>/<basename>
|
|
14
|
+
* ```
|
|
15
|
+
*
|
|
16
|
+
* CONTENT-ADDRESSED, and machine-wide rather than per-agent. Ten agents
|
|
17
|
+
* running the same 150MB whisper share one copy, and the hash in the path is
|
|
18
|
+
* what makes that safe: two files at the same address are the same bytes, so
|
|
19
|
+
* dedup and verification are the same fact.
|
|
20
|
+
*
|
|
21
|
+
* `AXON_MODELS_DIR` overrides the location. That seam is what lets a
|
|
22
|
+
* deployment image bake weights in at build time without the store, the
|
|
23
|
+
* kernel or the cognet knowing anything changed — the cognet only ever sees
|
|
24
|
+
* an absolute path.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
type ModelStoreOpts = {
|
|
28
|
+
/** Override the cache root. Tests point this at a scratch dir. */
|
|
29
|
+
root?: string
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export type StoredModel = {
|
|
33
|
+
/** Absolute path to the file — what a cognet ultimately receives. */
|
|
34
|
+
path: string
|
|
35
|
+
sha256: string
|
|
36
|
+
bytes: number
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function defaultRoot(): string {
|
|
40
|
+
return process.env.AXON_MODELS_DIR ?? join(homedir(), ".axon", "models")
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function ModelStore(opts: ModelStoreOpts = {}) {
|
|
44
|
+
const root = opts.root ?? defaultRoot()
|
|
45
|
+
/** specifier → { sha256, file }. A hint for unpinned models; see resolved(). */
|
|
46
|
+
const indexPath = join(root, "index.json")
|
|
47
|
+
|
|
48
|
+
/** Where a given hash's file lives. Pure — no I/O. */
|
|
49
|
+
function pathFor(sha256: string, filename: string): string {
|
|
50
|
+
return join(root, sha256, filename)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
return {
|
|
54
|
+
get root() {
|
|
55
|
+
return root
|
|
56
|
+
},
|
|
57
|
+
|
|
58
|
+
pathFor,
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The stored file for this model, or null when absent.
|
|
62
|
+
*
|
|
63
|
+
* Only answerable when the ref carries a hash — the address IS the
|
|
64
|
+
* hash, so without one there is nothing to look up. An unpinned model
|
|
65
|
+
* resolves through the fetch path, which learns the hash from the
|
|
66
|
+
* registry first.
|
|
67
|
+
*/
|
|
68
|
+
async find(model: ParsedModel): Promise<StoredModel | null> {
|
|
69
|
+
if (!model.sha256) return null
|
|
70
|
+
const path = pathFor(model.sha256, basenameOf(model))
|
|
71
|
+
if (!fsx.isFile(path)) return null
|
|
72
|
+
const info = await stat(path)
|
|
73
|
+
return { path, sha256: model.sha256, bytes: info.size }
|
|
74
|
+
},
|
|
75
|
+
|
|
76
|
+
/** True when these exact bytes are already cached. */
|
|
77
|
+
has(sha256: string, filename: string): boolean {
|
|
78
|
+
return fsx.isFile(pathFor(sha256, filename))
|
|
79
|
+
},
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* What a previous fetch resolved this specifier to, if anything.
|
|
83
|
+
*
|
|
84
|
+
* Content-addressing alone cannot answer for an UNPINNED model: the
|
|
85
|
+
* address is the hash, and without a declared hash there is nothing
|
|
86
|
+
* to look up. Registries do not reliably publish one either — HF's
|
|
87
|
+
* plain etag is a git blob sha, which is not the hash of the bytes it
|
|
88
|
+
* serves.
|
|
89
|
+
*
|
|
90
|
+
* So a small index maps specifier → hash, written after the first
|
|
91
|
+
* successful verified fetch. Without it, every unpinned model
|
|
92
|
+
* re-downloads on every prepare, which for a 150MB whisper is
|
|
93
|
+
* minutes of nothing.
|
|
94
|
+
*
|
|
95
|
+
* The index is a CACHE HINT, never authority: the file it points at
|
|
96
|
+
* is still content-addressed, so a corrupted or missing entry costs a
|
|
97
|
+
* refetch and cannot produce wrong bytes.
|
|
98
|
+
*/
|
|
99
|
+
async resolved(specifier: string): Promise<StoredModel | null> {
|
|
100
|
+
const index = await fsx.readJson<Record<string, { sha256: string; file: string }>>(indexPath)
|
|
101
|
+
const hit = index?.[specifier]
|
|
102
|
+
if (!hit) return null
|
|
103
|
+
|
|
104
|
+
const path = pathFor(hit.sha256, hit.file)
|
|
105
|
+
if (!fsx.isFile(path)) return null
|
|
106
|
+
|
|
107
|
+
const info = await stat(path)
|
|
108
|
+
return { path, sha256: hit.sha256, bytes: info.size }
|
|
109
|
+
},
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Every specifier this machine has fetched, with where it landed.
|
|
113
|
+
*
|
|
114
|
+
* The index already maps specifier → address for `resolved()`; this
|
|
115
|
+
* enumerates it, which is the question a surface listing "what is on
|
|
116
|
+
* this machine" asks. Derived from the same file rather than a second
|
|
117
|
+
* record, so a listing cannot disagree with a lookup.
|
|
118
|
+
*
|
|
119
|
+
* A specifier whose FILE has gone is omitted rather than reported:
|
|
120
|
+
* the index is a cache hint and the bytes are the fact, so an entry
|
|
121
|
+
* pointing at nothing is stale rather than a model.
|
|
122
|
+
*/
|
|
123
|
+
async list(): Promise<{ specifier: string; path: string; sha256: string; bytes: number }[]> {
|
|
124
|
+
const index = await fsx.readJson<Record<string, { sha256: string; file: string }>>(indexPath)
|
|
125
|
+
if (!index) return []
|
|
126
|
+
|
|
127
|
+
const found: { specifier: string; path: string; sha256: string; bytes: number }[] = []
|
|
128
|
+
for (const [specifier, hit] of Object.entries(index)) {
|
|
129
|
+
const path = pathFor(hit.sha256, hit.file)
|
|
130
|
+
if (!fsx.isFile(path)) continue
|
|
131
|
+
found.push({ specifier, path, sha256: hit.sha256, bytes: (await stat(path)).size })
|
|
132
|
+
}
|
|
133
|
+
return found
|
|
134
|
+
},
|
|
135
|
+
|
|
136
|
+
/** Record what a specifier resolved to, so the next run can skip the network. */
|
|
137
|
+
async remember(specifier: string, stored: StoredModel, filename: string): Promise<void> {
|
|
138
|
+
const index = (await fsx.readJson<Record<string, { sha256: string; file: string }>>(indexPath)) ?? {}
|
|
139
|
+
index[specifier] = { sha256: stored.sha256, file: filename }
|
|
140
|
+
await mkdir(root, { recursive: true })
|
|
141
|
+
await Bun.write(indexPath, JSON.stringify(index, null, 4))
|
|
142
|
+
},
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Commit downloaded bytes to the store.
|
|
146
|
+
*
|
|
147
|
+
* Hashes first, then writes to a temp path, then renames into place.
|
|
148
|
+
* A partial file must NEVER sit at a valid address: rename is atomic
|
|
149
|
+
* within a filesystem, so a reader either sees the whole file or no
|
|
150
|
+
* file. Without that, an interrupted 150MB download leaves a
|
|
151
|
+
* truncated model that every later run trusts forever — and
|
|
152
|
+
* content-addressing would be a lie, since the path would claim a
|
|
153
|
+
* hash the contents do not have.
|
|
154
|
+
*
|
|
155
|
+
* `expected` is the caller's claim about what these bytes should be
|
|
156
|
+
* (from a pin, or from the registry's own metadata). A mismatch
|
|
157
|
+
* throws rather than storing: caching corrupt weights is worse than
|
|
158
|
+
* failing to cache.
|
|
159
|
+
*/
|
|
160
|
+
async put(filename: string, data: Uint8Array, expected?: string): Promise<StoredModel> {
|
|
161
|
+
const sha256 = createHash("sha256").update(data).digest("hex")
|
|
162
|
+
|
|
163
|
+
if (expected && expected !== sha256) {
|
|
164
|
+
throw err("MODEL_HASH_MISMATCH", {
|
|
165
|
+
detail:
|
|
166
|
+
`${filename}: expected sha256 ${expected} but the downloaded bytes hash to ${sha256} `
|
|
167
|
+
+ `— the upstream file changed, or the download was corrupted`,
|
|
168
|
+
context: { filename, expected, actual: sha256 },
|
|
169
|
+
})
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const target = pathFor(sha256, filename)
|
|
173
|
+
if (fsx.isFile(target)) {
|
|
174
|
+
// Already cached by another agent, or a previous run. Same
|
|
175
|
+
// hash means the same bytes, so there is nothing to do.
|
|
176
|
+
return { path: target, sha256, bytes: data.byteLength }
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const staging = await mkdtemp(join(tmpdir(), "axon-model-"))
|
|
180
|
+
try {
|
|
181
|
+
const temp = join(staging, filename)
|
|
182
|
+
await Bun.write(temp, data)
|
|
183
|
+
await mkdir(join(target, ".."), { recursive: true })
|
|
184
|
+
await rename(temp, target)
|
|
185
|
+
} catch (cause) {
|
|
186
|
+
// A cross-device rename fails (EXDEV) when tmp and the store
|
|
187
|
+
// are on different filesystems — copy is the honest fallback,
|
|
188
|
+
// and Bun.write to the final path is still preferable to
|
|
189
|
+
// leaving nothing.
|
|
190
|
+
await mkdir(join(target, ".."), { recursive: true })
|
|
191
|
+
await Bun.write(target, data)
|
|
192
|
+
void cause
|
|
193
|
+
} finally {
|
|
194
|
+
await rm(staging, { recursive: true, force: true })
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
return { path: target, sha256, bytes: data.byteLength }
|
|
198
|
+
},
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export type ModelStoreT = ReturnType<typeof ModelStore>
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { createHash } from "node:crypto"
|
|
2
|
+
import { join } from "node:path"
|
|
3
|
+
import { fsx } from "../../../utils/fs"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A source module's content hash — its version identity.
|
|
7
|
+
*
|
|
8
|
+
* Covers exactly what a consumer of the module would see: its declaration
|
|
9
|
+
* (module.config.ts), its manifest (package.json), and its source tree. Build
|
|
10
|
+
* output and node_modules are deliberately excluded — they are derived, and
|
|
11
|
+
* hashing them would make the identity change without the module changing.
|
|
12
|
+
*
|
|
13
|
+
* Files are folded in sorted order (fsx.walk sorts), with an explicit
|
|
14
|
+
* name/content separator, so the digest is stable across machines and cannot
|
|
15
|
+
* be collided by moving content between files.
|
|
16
|
+
*/
|
|
17
|
+
export async function hashModule(root: string): Promise<string> {
|
|
18
|
+
const hash = createHash("sha256")
|
|
19
|
+
|
|
20
|
+
for (const file of ["module.config.ts", "package.json"]) {
|
|
21
|
+
const text = await fsx.readText(join(root, file))
|
|
22
|
+
if (text !== null) hash.update(`${file}\0${text}`)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
for (const { relPath, absPath } of await fsx.walk(join(root, "src"))) {
|
|
26
|
+
const text = await fsx.readText(absPath)
|
|
27
|
+
if (text !== null) hash.update(`src/${relPath}\0${text}`)
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return hash.digest("hex")
|
|
31
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
// modules — local source modules hard-imported by an agent's config.
|
|
2
|
+
// SourceModules() is the module's single entry point.
|
|
3
|
+
|
|
4
|
+
export { SourceModules, type SourceModulesT, type SourceModule, type LinkResult } from "./modules"
|
|
5
|
+
export type { LockT, LockFile, LockEntry } from "./lock"
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { writeFile } from "node:fs/promises"
|
|
2
|
+
import { fsx } from "../../../utils/fs"
|
|
3
|
+
import { Frame } from "../../frame"
|
|
4
|
+
|
|
5
|
+
export type LockEntry = {
|
|
6
|
+
/** Absolute path to the module root at prepare time (machine-local, informational only). */
|
|
7
|
+
sourcePath: string
|
|
8
|
+
/** SHA-256 hex digest over module.config.ts + package.json + src/. */
|
|
9
|
+
contentHash: string
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export type LockFile = {
|
|
13
|
+
lockfileVersion: 1
|
|
14
|
+
modules: Record<string, LockEntry>
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const FILE = "source-modules.lock.json"
|
|
18
|
+
|
|
19
|
+
type LockOpts = {
|
|
20
|
+
root: string
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Lock — .agent/cache/source-modules.lock.json.
|
|
25
|
+
*
|
|
26
|
+
* A source module has no semver of its own, so its version identity is a
|
|
27
|
+
* content hash of what was on disk at prepare time. This records that, which
|
|
28
|
+
* is what lets a deployment later answer "which cut of this local module
|
|
29
|
+
* shipped" and lets prepare answer "did it change since last time".
|
|
30
|
+
*
|
|
31
|
+
* Keyed by SHORT name (the name the blueprint scanner uses), not by package
|
|
32
|
+
* name — callers pruning by package name must map across, which prune() does.
|
|
33
|
+
*/
|
|
34
|
+
export function Lock(opts: LockOpts) {
|
|
35
|
+
// The lock is a cache in the strict sense — it records what prepare
|
|
36
|
+
// already did so the next prepare can skip it — so it lives with the
|
|
37
|
+
// other caches rather than loose in the frame root.
|
|
38
|
+
const frame = Frame({ root: opts.root, kind: "agent" })
|
|
39
|
+
const path = frame.file("cache", FILE)
|
|
40
|
+
|
|
41
|
+
async function read(): Promise<LockFile> {
|
|
42
|
+
return (await fsx.readJson<LockFile>(path)) ?? { lockfileVersion: 1, modules: {} }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async function write(lock: LockFile): Promise<void> {
|
|
46
|
+
frame.ensure("cache")
|
|
47
|
+
await writeFile(path, JSON.stringify(lock, null, 2) + "\n")
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
return {
|
|
51
|
+
path: path,
|
|
52
|
+
read: read,
|
|
53
|
+
|
|
54
|
+
/** The hash recorded for this module at the last prepare, if any. */
|
|
55
|
+
async recorded(name: string): Promise<string | undefined> {
|
|
56
|
+
return (await read()).modules[name]?.contentHash
|
|
57
|
+
},
|
|
58
|
+
|
|
59
|
+
/** Record a module's current identity. */
|
|
60
|
+
async record(name: string, entry: LockEntry): Promise<void> {
|
|
61
|
+
const lock = await read()
|
|
62
|
+
lock.modules[name] = entry
|
|
63
|
+
await write(lock)
|
|
64
|
+
},
|
|
65
|
+
|
|
66
|
+
/** Drop entries by short name. */
|
|
67
|
+
async forget(names: string[]): Promise<void> {
|
|
68
|
+
const lock = await read()
|
|
69
|
+
const dead = new Set(names)
|
|
70
|
+
for (const key of Object.keys(lock.modules)) {
|
|
71
|
+
if (dead.has(key)) delete lock.modules[key]
|
|
72
|
+
}
|
|
73
|
+
await write(lock)
|
|
74
|
+
},
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export type LockT = ReturnType<typeof Lock>
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { dirname, join, resolve } from "node:path"
|
|
2
|
+
import { fsx } from "../../../utils/fs"
|
|
3
|
+
import type { ManifestT } from "../manifest"
|
|
4
|
+
import { hashModule } from "./hash"
|
|
5
|
+
import { Lock } from "./lock"
|
|
6
|
+
|
|
7
|
+
/** One declared source module: its root on disk and the names it is known by. */
|
|
8
|
+
export type SourceModule = {
|
|
9
|
+
/** Short name, e.g. "fs" — what the blueprint scanner keys modules by. */
|
|
10
|
+
name: string
|
|
11
|
+
/** Full package name, e.g. "@axon/fs" — how package.json refers to it. */
|
|
12
|
+
packageName: string
|
|
13
|
+
root: string
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export type LinkResult = {
|
|
17
|
+
/** Whether the module's content changed since the last prepare. */
|
|
18
|
+
status: "linked" | "unchanged"
|
|
19
|
+
name: string
|
|
20
|
+
contentHash: string
|
|
21
|
+
/** Whether package.json gained or changed this module's `file:` entry — the caller's signal that a real install is needed. */
|
|
22
|
+
dependenciesChanged: boolean
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
type SourceModulesOpts = {
|
|
26
|
+
root: string
|
|
27
|
+
/** The agent's declaration files — the one owner of its dependency map. */
|
|
28
|
+
manifest: ManifestT
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* SourceModules — local modules hard-imported by the agent's config.
|
|
33
|
+
*
|
|
34
|
+
* Declared via `defineModule()` in axon.config.ts (hostDir/../mods/x/module.config.ts,
|
|
35
|
+
* Nuxt-style) rather than pulled from the registry. They have no semver of
|
|
36
|
+
* their own: their identity is a content hash of what is on disk, recorded in
|
|
37
|
+
* the lock at prepare time.
|
|
38
|
+
*
|
|
39
|
+
* Their DEPENDENCIES are Bun's problem, not ours. A source module is declared
|
|
40
|
+
* as a `file:` dependency in the agent's package.json, so Bun links it and
|
|
41
|
+
* installs its dependencies exactly as it would for a published package. That
|
|
42
|
+
* replaced a hand-rolled dependency merge which spliced a module's deps into
|
|
43
|
+
* the agent's manifest and threw when two modules declared incompatible ranges
|
|
44
|
+
* — a conflict Bun resolves by nesting. Registry and source modules now
|
|
45
|
+
* resolve through one mechanism, which is what "a module's origin must never
|
|
46
|
+
* change how its deps resolve" was always trying to say.
|
|
47
|
+
*/
|
|
48
|
+
export function SourceModules(opts: SourceModulesOpts) {
|
|
49
|
+
const root = opts.root
|
|
50
|
+
const manifest = opts.manifest
|
|
51
|
+
const lock = Lock({ root: root })
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Point package.json at the module on disk.
|
|
55
|
+
*
|
|
56
|
+
* The path is ABSOLUTE. A relative `file:` specifier resolves against the
|
|
57
|
+
* manifest's own location, and Tree.install() deliberately copies the
|
|
58
|
+
* manifest into a temp directory to escape a surrounding monorepo workspace
|
|
59
|
+
* — at which point "../mods/x" names nothing. Source modules are by
|
|
60
|
+
* definition local and unpublished, so a machine-specific path costs
|
|
61
|
+
* nothing: a fresh clone re-derives it on the next prepare.
|
|
62
|
+
*/
|
|
63
|
+
function specifierFor(module: SourceModule): string {
|
|
64
|
+
return `file:${toPosix(resolve(module.root))}`
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return {
|
|
68
|
+
lock: lock,
|
|
69
|
+
|
|
70
|
+
/** Derive a module's root, short name, and package name from its module.config.ts path. */
|
|
71
|
+
async resolve(configPath: string): Promise<SourceModule> {
|
|
72
|
+
const moduleRoot = dirname(configPath)
|
|
73
|
+
const text = await fsx.readText(join(moduleRoot, "package.json"))
|
|
74
|
+
const declared = text ? (JSON.parse(text) as { name?: string }).name : undefined
|
|
75
|
+
const packageName = declared ?? basename(moduleRoot)
|
|
76
|
+
|
|
77
|
+
return {
|
|
78
|
+
name: packageName.includes("/") ? packageName.split("/").pop()! : packageName,
|
|
79
|
+
packageName: packageName,
|
|
80
|
+
root: moduleRoot,
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Declare a module as a `file:` dependency and record its content hash.
|
|
86
|
+
* Returns "unchanged" when the hash matches the last prepare.
|
|
87
|
+
*/
|
|
88
|
+
async link(module: SourceModule): Promise<LinkResult> {
|
|
89
|
+
const dependenciesChanged = await manifest.package.dependencies.set({
|
|
90
|
+
[module.packageName]: specifierFor(module),
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
const contentHash = await hashModule(module.root)
|
|
94
|
+
const previous = await lock.recorded(module.name)
|
|
95
|
+
await lock.record(module.name, { sourcePath: module.root, contentHash: contentHash })
|
|
96
|
+
|
|
97
|
+
return {
|
|
98
|
+
status: previous === contentHash ? "unchanged" : "linked",
|
|
99
|
+
name: module.name,
|
|
100
|
+
contentHash: contentHash,
|
|
101
|
+
dependenciesChanged: dependenciesChanged,
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Drop `file:` dependencies for modules the config no longer declares,
|
|
107
|
+
* and their stale lock entries. Returns the removed package names.
|
|
108
|
+
*
|
|
109
|
+
* prepare() only ever added: removing a `defineModule()` import left its
|
|
110
|
+
* `file:` dependency behind forever, so Bun kept linking the local
|
|
111
|
+
* directory and installing the same module from the registry could
|
|
112
|
+
* never replace it. The config is the declaration — a `file:` dep with
|
|
113
|
+
* nothing declaring it is dead.
|
|
114
|
+
*
|
|
115
|
+
* Only `file:` deps are considered. A registry module is removed by
|
|
116
|
+
* `axon uninstall`, and the agent's own npm dependencies are the
|
|
117
|
+
* author's.
|
|
118
|
+
*
|
|
119
|
+
* The FRAMEWORK is exempt. It is legitimately `file:` when an agent
|
|
120
|
+
* points at a local checkout — how the framework is developed, and
|
|
121
|
+
* how the test suite links the tree under test — and it is not a
|
|
122
|
+
* source module, so this would prune it on the very prepare that
|
|
123
|
+
* declared it. The symptom is a manifest with `overrides` and no
|
|
124
|
+
* `@arcforge/*` dependencies, then "cognet is not installed".
|
|
125
|
+
*/
|
|
126
|
+
async prune(declared: SourceModule[]): Promise<string[]> {
|
|
127
|
+
const live = new Set(declared.map(module => module.packageName))
|
|
128
|
+
const dependencies = await manifest.package.dependencies.all()
|
|
129
|
+
|
|
130
|
+
const dead = Object.entries(dependencies)
|
|
131
|
+
.filter(([name, specifier]) =>
|
|
132
|
+
specifier.startsWith("file:")
|
|
133
|
+
&& !live.has(name)
|
|
134
|
+
&& !name.startsWith("@arcforge/"))
|
|
135
|
+
.map(([name]) => name)
|
|
136
|
+
if (dead.length === 0) return []
|
|
137
|
+
|
|
138
|
+
const removed = await manifest.package.dependencies.remove(dead)
|
|
139
|
+
// The lock is keyed by short name, not package name.
|
|
140
|
+
await lock.forget(removed.map(name => (name.includes("/") ? name.split("/").pop()! : name)))
|
|
141
|
+
|
|
142
|
+
return removed
|
|
143
|
+
},
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export type SourceModulesT = ReturnType<typeof SourceModules>
|
|
148
|
+
|
|
149
|
+
function basename(path: string): string {
|
|
150
|
+
return path.split("/").pop()!
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function toPosix(path: string): string {
|
|
154
|
+
return path.split(/[\\/]/).join("/")
|
|
155
|
+
}
|