@augurworks/augur 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CANVAS.md +858 -0
- package/INSTALL.md +603 -0
- package/LICENSE +27 -0
- package/README.md +150 -0
- package/TRADEMARKS.md +72 -0
- package/agents/README.md +65 -0
- package/agents/canon-extract.md +117 -0
- package/agents/canon.md +151 -0
- package/agents/canvas.md +88 -0
- package/agents/currency.md +74 -0
- package/agents/drafts.md +92 -0
- package/agents/identity.md +148 -0
- package/agents/prototype-contract.md +69 -0
- package/agents/publishing.md +324 -0
- package/agents/review-feedback.md +76 -0
- package/agents/space-json.md +132 -0
- package/agents/ui-skill.md +101 -0
- package/agents/working-marks.md +86 -0
- package/brand/augur-eye.svg +3 -0
- package/brand/augur-icon.svg +41 -0
- package/brand/augur-mark.svg +5 -0
- package/brand/lentianova-bulky.otf +0 -0
- package/brand/lentianova-median.otf +0 -0
- package/brand/lentianova.woff2 +0 -0
- package/build.js +8409 -0
- package/changelog.md +133 -0
- package/fonts/inter-latin-wght-normal.woff2 +0 -0
- package/package.json +57 -0
- package/pitis/PETS.md +121 -0
- package/pitis/README.md +117 -0
- package/pitis/index.html +266 -0
- package/pitis/piti.build.js +81 -0
- package/pitis/piti.js +929 -0
- package/pitis/roast-agent.md +185 -0
- package/realtime/src/index.js +119 -0
- package/realtime/wrangler.example.toml +44 -0
- package/scripts/adopt.mjs +106 -0
- package/scripts/board-snapshot.mjs +197 -0
- package/scripts/build-seed-pack.mjs +33 -0
- package/scripts/bundle-rekey.mjs +112 -0
- package/scripts/bundle-tenancy-rehearsal.mjs +781 -0
- package/scripts/canon-extract.mjs +578 -0
- package/scripts/canon.mjs +599 -0
- package/scripts/canvas-screen.mjs +333 -0
- package/scripts/check-local.mjs +136 -0
- package/scripts/clawd-art.mjs +75 -0
- package/scripts/clawd-canvas.mjs +741 -0
- package/scripts/cli.mjs +100 -0
- package/scripts/clone.mjs +422 -0
- package/scripts/close.mjs +30 -0
- package/scripts/connect.mjs +103 -0
- package/scripts/deploy.mjs +157 -0
- package/scripts/dev.mjs +54 -0
- package/scripts/doc-lint.mjs +143 -0
- package/scripts/export.mjs +300 -0
- package/scripts/fork.mjs +64 -0
- package/scripts/freeze.mjs +74 -0
- package/scripts/frontdoor-parity.mjs +180 -0
- package/scripts/hook.mjs +67 -0
- package/scripts/identity-rekey.mjs +112 -0
- package/scripts/init.mjs +122 -0
- package/scripts/land.mjs +40 -0
- package/scripts/lib/adapters.mjs +156 -0
- package/scripts/lib/board-snapshot.mjs +511 -0
- package/scripts/lib/draft.mjs +385 -0
- package/scripts/lib/graduate.mjs +276 -0
- package/scripts/lib/instance.mjs +74 -0
- package/scripts/lib/marks.mjs +107 -0
- package/scripts/lib/materialize.mjs +138 -0
- package/scripts/lib/merge3.mjs +124 -0
- package/scripts/lib/offline-posture.mjs +120 -0
- package/scripts/lib/offline-respawn.mjs +18 -0
- package/scripts/lib/offline-wrangler.mjs +78 -0
- package/scripts/lib/publish-compose.mjs +12 -0
- package/scripts/lib/publish-conflict.mjs +78 -0
- package/scripts/lib/publish-evidence.mjs +144 -0
- package/scripts/lib/refine-compare.mjs +140 -0
- package/scripts/lib/refine-ledger.mjs +150 -0
- package/scripts/lib/refine-png.mjs +149 -0
- package/scripts/lib/seed-pack-build.mjs +141 -0
- package/scripts/lib/state-compare.mjs +68 -0
- package/scripts/lib/store.mjs +122 -0
- package/scripts/login.mjs +84 -0
- package/scripts/mark.mjs +112 -0
- package/scripts/migrate-board-keys.mjs +164 -0
- package/scripts/migrate.mjs +265 -0
- package/scripts/no-foreign-vocabulary.mjs +216 -0
- package/scripts/no-product-names.mjs +157 -0
- package/scripts/no-tenant-globals.mjs +1134 -0
- package/scripts/offline.mjs +247 -0
- package/scripts/og.mjs +170 -0
- package/scripts/one-tenant-resolver.mjs +146 -0
- package/scripts/open.mjs +63 -0
- package/scripts/publish.mjs +1041 -0
- package/scripts/read.mjs +31 -0
- package/scripts/refine.mjs +455 -0
- package/scripts/release-drift.mjs +138 -0
- package/scripts/restore.mjs +322 -0
- package/scripts/review.mjs +125 -0
- package/scripts/save.mjs +21 -0
- package/scripts/shell-lint.mjs +189 -0
- package/scripts/ship.mjs +460 -0
- package/scripts/shoot.mjs +264 -0
- package/scripts/smoke-invite.mjs +113 -0
- package/scripts/state-inventory.mjs +223 -0
- package/scripts/status.mjs +177 -0
- package/scripts/sync.mjs +27 -0
- package/scripts/tenant-do-rehearsal.mjs +1314 -0
- package/scripts/ui-copy-lint.mjs +202 -0
- package/scripts/watch.mjs +32 -0
- package/scripts/wrangler-preflight.mjs +246 -0
- package/seed/CANON.md +43 -0
- package/seed/README.md +119 -0
- package/seed/prototype-status.json +9 -0
- package/seed/registry.json +88 -0
- package/seed/skills/starter-ui/SKILL.md +52 -0
- package/seed/skills/starter-ui/skill.json +11 -0
- package/seed/skills/starter-ui/starter-tokens.css +73 -0
- package/seed/skills/starter-ui/starter-ui.css +318 -0
- package/seed/skills/starter-ui/starter-ui.js +50 -0
- package/seed/space.json +5 -0
- package/seed/start-here/prototypes/connect-your-terminal/index.html +428 -0
- package/seed/start-here/prototypes/connect-your-terminal/preview.webp +0 -0
- package/seed/start-here/prototypes/sample-with-comments/index.html +172 -0
- package/seed/start-here/prototypes/sample-with-comments/preview.webp +0 -0
- package/seed/start-here/prototypes/set-up-your-design-system/index.html +353 -0
- package/seed/start-here/prototypes/set-up-your-design-system/preview.webp +0 -0
- package/seed/threads.json +52 -0
- package/seed/worked-examples/prototypes/field-readings/index.html +273 -0
- package/seed/worked-examples/prototypes/field-readings/preview.webp +0 -0
- package/seed/worked-examples/prototypes/slide-deck/index.html +214 -0
- package/seed/worked-examples/prototypes/slide-deck/preview.webp +0 -0
- package/seed/worked-examples/prototypes/specimen-viewer/index.html +203 -0
- package/seed/worked-examples/prototypes/specimen-viewer/preview.webp +0 -0
- package/src/_worker.js +12668 -0
- package/src/board-key.mjs +57 -0
- package/src/board-room.mjs +881 -0
- package/src/bundle-keys.mjs +138 -0
- package/src/canon/collect-in-browser.js +215 -0
- package/src/canon/emit.mjs +232 -0
- package/src/canon/observe.mjs +260 -0
- package/src/canon/schema.mjs +451 -0
- package/src/canvas/DSEG-LICENSE.txt +95 -0
- package/src/canvas/DSEG7Classic-Bold.woff2 +0 -0
- package/src/canvas/canvas.css +885 -0
- package/src/canvas/canvas.js +5780 -0
- package/src/canvas/capture.js +485 -0
- package/src/chrome/appchrome.mjs +568 -0
- package/src/currency.mjs +203 -0
- package/src/delete-confirmation.mjs +254 -0
- package/src/drafts/drafts.js +326 -0
- package/src/entry.js +58 -0
- package/src/galleries.mjs +400 -0
- package/src/health-cron.mjs +173 -0
- package/src/identity.json +1 -0
- package/src/kv-codec.mjs +135 -0
- package/src/kv-identity.mjs +259 -0
- package/src/mail.mjs +599 -0
- package/src/provenance.mjs +74 -0
- package/src/publish-compose.mjs +363 -0
- package/src/publish-fork.mjs +221 -0
- package/src/publish-units.mjs +44 -0
- package/src/purge.mjs +81 -0
- package/src/review/COMMENTING-UX.md +306 -0
- package/src/review/cat.png +0 -0
- package/src/review/comment-cursor.svg +7 -0
- package/src/review/comments.js +1738 -0
- package/src/room-ticket.mjs +96 -0
- package/src/roster-chip.mjs +32 -0
- package/src/seed-pack.mjs +261 -0
- package/src/state-inventory.mjs +249 -0
- package/src/sw-logic.mjs +48 -0
- package/src/tenant-cache.mjs +141 -0
- package/src/tenant-context.mjs +386 -0
- package/src/tenant-do.js +2940 -0
- package/src/tenant-host.mjs +144 -0
- package/src/tenant-quotas.mjs +140 -0
- package/src/unit-core.mjs +103 -0
- package/src/unit-object.mjs +338 -0
- package/templates/README.md +67 -0
- package/templates/shell/deploy.yml +145 -0
- package/templates/shell/engine-bump.yml +142 -0
- package/templates/shell/health.yml +233 -0
- package/templates/shell/kv-backup.yml +201 -0
- package/templates/shell/roster-update.yml +110 -0
- package/templates/shell/space-preflight.yml +35 -0
- package/templates/shell/space-rebake.yml +79 -0
- package/templates/shell/store-backup.yml +152 -0
- package/templates/shell/wrangler.example.toml +266 -0
- package/templates/space/publish.yml +91 -0
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
// deploy.mjs — first-class local deploy: build → direct-upload → URL on stdout.
|
|
2
|
+
//
|
|
3
|
+
// npm run deploy build all spaces and ship them to the instance's
|
|
4
|
+
// Pages project (direct upload, hash-incremental)
|
|
5
|
+
// npm run deploy -- --check build only; print what would ship and exit
|
|
6
|
+
//
|
|
7
|
+
// Contract (agents rely on this): synchronous, zero prompts, exit code = truth,
|
|
8
|
+
// the deployment URL is the LAST line on stdout; progress goes to stderr. Auth
|
|
9
|
+
// and instance identity come from .env.deploy (see .env.deploy.example) — explicit
|
|
10
|
+
// env always wins, then the sibling deploy shell resolved by shape.
|
|
11
|
+
//
|
|
12
|
+
// Provenance: each repo that contributes to the build (engine + every space) is
|
|
13
|
+
// stamped into /_build.json with its sha; a repo with uncommitted changes gains
|
|
14
|
+
// `"dirty": true` so a working-tree ship is visible, never hidden (build.js merges
|
|
15
|
+
// GV_BUILD_DIRTY_JSON — computed here, since only a local deploy can be dirty).
|
|
16
|
+
|
|
17
|
+
import { spawn } from "node:child_process";
|
|
18
|
+
import { execFileSync } from "node:child_process";
|
|
19
|
+
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import { fileURLToPath } from "node:url";
|
|
22
|
+
import { findShellDir } from "./lib/instance.mjs";
|
|
23
|
+
|
|
24
|
+
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
25
|
+
const CHECK = process.argv.includes("--check");
|
|
26
|
+
const PREVIEW = process.argv.includes("--preview");
|
|
27
|
+
const log = (msg) => console.error(`\x1b[36m[deploy]\x1b[0m ${msg}`);
|
|
28
|
+
|
|
29
|
+
function readEnvFile(p) {
|
|
30
|
+
const out = {};
|
|
31
|
+
try {
|
|
32
|
+
for (const line of readFileSync(p, "utf8").split("\n")) {
|
|
33
|
+
const m = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*)\s*$/);
|
|
34
|
+
if (m) out[m[1]] = m[2].trim();
|
|
35
|
+
}
|
|
36
|
+
} catch {}
|
|
37
|
+
return out;
|
|
38
|
+
}
|
|
39
|
+
const DEPLOY_ENV = readEnvFile(path.join(ROOT, ".env.deploy"));
|
|
40
|
+
for (const [k, v] of Object.entries(DEPLOY_ENV)) if (!(k in process.env)) process.env[k] = v;
|
|
41
|
+
|
|
42
|
+
// Same space discovery as offline.mjs: canonical sibling clones (edit-here) win;
|
|
43
|
+
// a lone engine clone falls back to the pinned ./spaces mounts.
|
|
44
|
+
const PARENT = path.join(ROOT, "..");
|
|
45
|
+
let siblingSpaces = [];
|
|
46
|
+
try {
|
|
47
|
+
siblingSpaces = readdirSync(PARENT, { withFileTypes: true })
|
|
48
|
+
.filter((e) => e.isDirectory() && !e.name.startsWith(".")
|
|
49
|
+
&& existsSync(path.join(PARENT, e.name, "space.json")))
|
|
50
|
+
.map((e) => path.join(PARENT, e.name));
|
|
51
|
+
} catch {}
|
|
52
|
+
const usingSiblings = siblingSpaces.length > 0;
|
|
53
|
+
const SPACES_ROOT = process.env.GV_SPACES_ROOT || (usingSiblings ? PARENT : path.join(ROOT, "spaces"));
|
|
54
|
+
|
|
55
|
+
const SHELL_DIR = findShellDir(ROOT);
|
|
56
|
+
const IDENTITY_PATH = process.env.GV_IDENTITY_PATH
|
|
57
|
+
|| [SHELL_DIR && path.join(SHELL_DIR, "identity.json")].filter(Boolean).find((p) => existsSync(p));
|
|
58
|
+
const DEPLOY_CONFIG_PATH = process.env.GV_DEPLOY_CONFIG_PATH
|
|
59
|
+
|| (SHELL_DIR && existsSync(path.join(SHELL_DIR, "deploy.config.json"))
|
|
60
|
+
? path.join(SHELL_DIR, "deploy.config.json") : null);
|
|
61
|
+
if (!IDENTITY_PATH) {
|
|
62
|
+
log("no identity.json found (GV_IDENTITY_PATH unset, no sibling deploy shell) — refusing to ship an open-gated site.");
|
|
63
|
+
process.exit(1);
|
|
64
|
+
}
|
|
65
|
+
if (!CHECK && !process.env.PAGES_PROJECT) {
|
|
66
|
+
log("PAGES_PROJECT unset — set it in .env.deploy (the instance's Pages project name).");
|
|
67
|
+
process.exit(1);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Dirty provenance: engine + each space root; `git status --porcelain` non-empty → dirty.
|
|
71
|
+
function repoState(dir) {
|
|
72
|
+
const out = { sha: null, dirty: false };
|
|
73
|
+
try {
|
|
74
|
+
out.sha = execFileSync("git", ["-C", dir, "rev-parse", "HEAD"],
|
|
75
|
+
{ encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim();
|
|
76
|
+
out.dirty = execFileSync("git", ["-C", dir, "status", "--porcelain"],
|
|
77
|
+
{ encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim().length > 0;
|
|
78
|
+
} catch {}
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
81
|
+
const spaceDirs = usingSiblings ? siblingSpaces : (() => {
|
|
82
|
+
try {
|
|
83
|
+
return readdirSync(SPACES_ROOT, { withFileTypes: true })
|
|
84
|
+
.filter((e) => e.isDirectory() && existsSync(path.join(SPACES_ROOT, e.name, "space.json")))
|
|
85
|
+
.map((e) => path.join(SPACES_ROOT, e.name));
|
|
86
|
+
} catch { return []; }
|
|
87
|
+
})();
|
|
88
|
+
const dirty = { engine: repoState(ROOT).dirty };
|
|
89
|
+
const states = { engine: repoState(ROOT) };
|
|
90
|
+
for (const dir of spaceDirs) {
|
|
91
|
+
let id = path.basename(dir);
|
|
92
|
+
try { id = JSON.parse(readFileSync(path.join(dir, "space.json"), "utf8")).id || id; } catch {}
|
|
93
|
+
states[id] = repoState(dir);
|
|
94
|
+
dirty[id] = states[id].dirty;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// ── build ────────────────────────────────────────────────────────────────────
|
|
98
|
+
const BUILD_ENV = {
|
|
99
|
+
...process.env,
|
|
100
|
+
GV_SPACES_ROOT: SPACES_ROOT,
|
|
101
|
+
GV_IDENTITY_PATH: IDENTITY_PATH,
|
|
102
|
+
...(DEPLOY_CONFIG_PATH ? { GV_DEPLOY_CONFIG_PATH: DEPLOY_CONFIG_PATH } : {}),
|
|
103
|
+
GV_BUILD_DIRTY_JSON: JSON.stringify(dirty),
|
|
104
|
+
};
|
|
105
|
+
log(`building from ${usingSiblings ? "sibling space clones" : SPACES_ROOT}`);
|
|
106
|
+
const started = Date.now();
|
|
107
|
+
const buildCode = await new Promise((resolve) => {
|
|
108
|
+
const proc = spawn("node", ["build.js"], { cwd: ROOT, env: BUILD_ENV, stdio: ["ignore", 2, 2] });
|
|
109
|
+
proc.on("close", resolve);
|
|
110
|
+
});
|
|
111
|
+
if (buildCode !== 0) { log(`build failed (exit ${buildCode})`); process.exit(buildCode || 1); }
|
|
112
|
+
log(`built in ${((Date.now() - started) / 1000).toFixed(1)}s`);
|
|
113
|
+
|
|
114
|
+
for (const [id, s] of Object.entries(states))
|
|
115
|
+
log(` ${id}: ${s.sha ? s.sha.slice(0, 9) : "no-git"}${s.dirty ? " \x1b[33m(dirty — working tree ships)\x1b[0m" : ""}`);
|
|
116
|
+
|
|
117
|
+
if (CHECK) {
|
|
118
|
+
// Summary only: per-space file counts from dist, then stop before any upload.
|
|
119
|
+
const dist = path.join(ROOT, "dist");
|
|
120
|
+
const count = (dir) => {
|
|
121
|
+
let n = 0;
|
|
122
|
+
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
|
123
|
+
if (e.isDirectory()) n += count(path.join(dir, e.name));
|
|
124
|
+
else n += 1;
|
|
125
|
+
}
|
|
126
|
+
return n;
|
|
127
|
+
};
|
|
128
|
+
log(`--check: dist holds ${count(dist)} files; nothing shipped.`);
|
|
129
|
+
console.log("(check-only, no deployment)");
|
|
130
|
+
process.exit(0);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// ── direct upload (hash-incremental; wrangler prompts suppressed) ────────────
|
|
134
|
+
// --preview ships to a non-production branch: full end-to-end upload, own URL,
|
|
135
|
+
// production untouched — the safe default for verification runs.
|
|
136
|
+
const args = ["--yes", "wrangler", "pages", "deploy", "dist",
|
|
137
|
+
"--project-name", process.env.PAGES_PROJECT,
|
|
138
|
+
"--branch", PREVIEW ? "preview" : "main", "--commit-dirty=true"];
|
|
139
|
+
const chunks = [];
|
|
140
|
+
const wranglerCode = await new Promise((resolve) => {
|
|
141
|
+
const proc = spawn("npx", args, { cwd: ROOT, env: process.env, stdio: ["ignore", "pipe", 2] });
|
|
142
|
+
proc.stdout.on("data", (d) => { chunks.push(d); process.stderr.write(d); });
|
|
143
|
+
proc.on("close", resolve);
|
|
144
|
+
});
|
|
145
|
+
if (wranglerCode !== 0) { log(`upload failed (exit ${wranglerCode})`); process.exit(wranglerCode || 1); }
|
|
146
|
+
|
|
147
|
+
// The deployment URL is the contract: last line on stdout. Prefer the instance's
|
|
148
|
+
// canonical origin (deploy.config.json siteOrigin) — that's where people look —
|
|
149
|
+
// falling back to the per-deploy URL wrangler prints.
|
|
150
|
+
let deployUrl = null;
|
|
151
|
+
const m = Buffer.concat(chunks).toString("utf8").match(/https:\/\/[^\s]+\.pages\.dev[^\s]*/g);
|
|
152
|
+
if (m) deployUrl = m[m.length - 1];
|
|
153
|
+
let siteOrigin = null;
|
|
154
|
+
try { siteOrigin = JSON.parse(readFileSync(DEPLOY_CONFIG_PATH, "utf8")).siteOrigin || null; } catch {}
|
|
155
|
+
log(`deployed in ${((Date.now() - started) / 1000).toFixed(1)}s total`);
|
|
156
|
+
// Preview deploys report their own URL; production reports the canonical origin.
|
|
157
|
+
console.log((PREVIEW ? deployUrl : siteOrigin || deployUrl) || "(deployed — no URL reported by wrangler)");
|
package/scripts/dev.mjs
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// dev.mjs — the standalone local shell: run the full Augur experience (login
|
|
2
|
+
// gate, rail, overlays, live reload) in ANY single space folder.
|
|
3
|
+
//
|
|
4
|
+
// cd <space repo> && augur dev (or node <engine>/scripts/dev.mjs)
|
|
5
|
+
//
|
|
6
|
+
// A thin wrapper over offline.mjs that fills in the standalone defaults:
|
|
7
|
+
// • the cwd carries a space.json → GV_SPACES_ROOT = cwd: a one-space site,
|
|
8
|
+
// built at the root URLs, exactly as a hosted default space would serve
|
|
9
|
+
// • no deploy shell resolves → a local dev identity (dev@local / password
|
|
10
|
+
// "dev", admin) so the gate and admin panel behave like a real instance
|
|
11
|
+
// instead of falling open — the "same shell as online" contract
|
|
12
|
+
// • KV stays local unless real credentials are present (offline.mjs's rule)
|
|
13
|
+
// From a multi-space workspace (sibling spaces + shell) it degrades gracefully to
|
|
14
|
+
// plain offline mode semantics for the cwd space.
|
|
15
|
+
|
|
16
|
+
import { spawn } from "node:child_process";
|
|
17
|
+
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
18
|
+
import os from "node:os";
|
|
19
|
+
import path from "node:path";
|
|
20
|
+
import { fileURLToPath } from "node:url";
|
|
21
|
+
import { findShellDir } from "./lib/instance.mjs";
|
|
22
|
+
import { __testables } from "../src/_worker.js";
|
|
23
|
+
|
|
24
|
+
const SCRIPTS = path.dirname(fileURLToPath(import.meta.url));
|
|
25
|
+
const ROOT = path.resolve(SCRIPTS, "..");
|
|
26
|
+
const log = (msg) => console.error(`\x1b[35m[dev]\x1b[0m ${msg}`);
|
|
27
|
+
|
|
28
|
+
const env = { ...process.env };
|
|
29
|
+
const cwd = process.cwd();
|
|
30
|
+
if (existsSync(path.join(cwd, "space.json")) && !env.GV_SPACES_ROOT) {
|
|
31
|
+
env.GV_SPACES_ROOT = cwd;
|
|
32
|
+
log(`single-space mode: ${cwd}`);
|
|
33
|
+
}
|
|
34
|
+
if (!env.GV_IDENTITY_PATH && !findShellDir(ROOT)) {
|
|
35
|
+
const dir = path.join(os.tmpdir(), "augur-dev");
|
|
36
|
+
mkdirSync(dir, { recursive: true });
|
|
37
|
+
const file = path.join(dir, "identity.json");
|
|
38
|
+
// A real pbkdf2$… hash, not a plaintext `pass` — verifyPassword() (src/_worker.js)
|
|
39
|
+
// accepts only that format, so a plaintext seed used to read as an "active" account
|
|
40
|
+
// (effectiveSecret resolves it) while being unable to ever log in (verifyPassword
|
|
41
|
+
// rejects it outright) — the documented dev@local / "dev" fallback 401ed, every time.
|
|
42
|
+
// Derive it the same way a redeemed invite does (__testables.hashPassword is the
|
|
43
|
+
// exact function invitePost calls), so the seeded account can actually authenticate.
|
|
44
|
+
const passHash = await __testables.hashPassword("dev");
|
|
45
|
+
writeFileSync(file, JSON.stringify([{
|
|
46
|
+
email: "dev@local", name: "Dev", passHash,
|
|
47
|
+
initials: "D", color: "#2c2150", role: "admin",
|
|
48
|
+
}], null, 2));
|
|
49
|
+
env.GV_IDENTITY_PATH = file;
|
|
50
|
+
log('no deploy shell found — local dev identity active (sign in: dev@local / "dev")');
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const child = spawn(process.execPath, [path.join(SCRIPTS, "offline.mjs")], { env, stdio: "inherit" });
|
|
54
|
+
child.on("close", (code) => process.exit(code ?? 0));
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// doc-lint — a doc-drift canary. Checks that .md files still tell the truth:
|
|
3
|
+
// a. backtick-quoted relative file paths exist on disk
|
|
4
|
+
// b. every `npm run <script>` mentioned exists in package.json
|
|
5
|
+
// c. relative markdown links (and #anchors) resolve
|
|
6
|
+
// Usage: node scripts/doc-lint.mjs [--repo <path>]... [--quiet]
|
|
7
|
+
// No config, no dependencies. Exit 1 on any failure.
|
|
8
|
+
import fs from "node:fs";
|
|
9
|
+
import path from "node:path";
|
|
10
|
+
import { spawnSync } from "node:child_process";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
|
|
13
|
+
// The canary guards the AGENT-CONTEXT layer (root docs, skills/<x>-ui/, agents/) — not
|
|
14
|
+
// scratch, archives, extracted research, prototype build notes, or source-embedded
|
|
15
|
+
// build notes, which carry their own noisy cross-refs (placeholder paths, throwaway
|
|
16
|
+
// links) and would swamp the signal. Recursion reaches the two-deep context docs
|
|
17
|
+
// (skills/<x>-ui/SKILL.md) while these trees stay out.
|
|
18
|
+
const SKIP_DIRS = new Set([
|
|
19
|
+
"node_modules", "dist", ".git", "spaces",
|
|
20
|
+
"playground", "prototypes", "docs", "references", "research", "context",
|
|
21
|
+
"src", "tracks", "toolkit", "img", "webapp-testing", "_extracted", "_archive",
|
|
22
|
+
]);
|
|
23
|
+
const skipDir = (name) => SKIP_DIRS.has(name) || /-exports$/.test(name); // *-exports: local capture bundles
|
|
24
|
+
const args = process.argv.slice(2);
|
|
25
|
+
const quiet = args.includes("--quiet");
|
|
26
|
+
const repos = [];
|
|
27
|
+
for (let i = 0; i < args.length; i++) if (args[i] === "--repo" && args[i + 1]) repos.push(path.resolve(args[++i]));
|
|
28
|
+
if (!repos.length) repos.push(path.dirname(path.dirname(fileURLToPath(import.meta.url))));
|
|
29
|
+
|
|
30
|
+
const failures = [];
|
|
31
|
+
const fail = (file, line, kind, ref) => failures.push(`${file}:${line}: ${kind}: ${ref}`);
|
|
32
|
+
|
|
33
|
+
// GitHub heading slug: lowercase, strip punctuation, spaces → dashes.
|
|
34
|
+
const slug = (s) =>
|
|
35
|
+
s.toLowerCase().trim()
|
|
36
|
+
.replace(/[^\p{L}\p{N}\s_-]/gu, "")
|
|
37
|
+
.replace(/\s+/g, "-");
|
|
38
|
+
|
|
39
|
+
function headingsOf(file) {
|
|
40
|
+
const out = new Set();
|
|
41
|
+
const counts = {};
|
|
42
|
+
for (const line of fs.readFileSync(file, "utf8").split("\n")) {
|
|
43
|
+
const m = /^#{1,6}\s+(.+?)\s*#*\s*$/.exec(line);
|
|
44
|
+
if (!m) continue;
|
|
45
|
+
let s = slug(m[1].replace(/`/g, ""));
|
|
46
|
+
if (counts[s] != null) s = `${s}-${++counts[s]}`;
|
|
47
|
+
else counts[s] = 0;
|
|
48
|
+
out.add(s);
|
|
49
|
+
}
|
|
50
|
+
return out;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Recurse the whole tree (skipping SKIP_DIRS, dotdirs, *-exports): the most-read docs
|
|
54
|
+
// live two levels down (e.g. skills/<x>-ui/SKILL.md) and a one-level scan never saw them.
|
|
55
|
+
function mdFiles(root) {
|
|
56
|
+
const out = [];
|
|
57
|
+
const walk = (dir) => {
|
|
58
|
+
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
59
|
+
if (e.isDirectory()) {
|
|
60
|
+
if (!skipDir(e.name) && !e.name.startsWith(".")) walk(path.join(dir, e.name));
|
|
61
|
+
} else if (e.isFile() && e.name.endsWith(".md")) {
|
|
62
|
+
out.push(path.join(dir, e.name));
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
walk(root);
|
|
67
|
+
return out;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// A backtick span counts as a checkable file path only if it is plausibly one:
|
|
71
|
+
// has a slash, no placeholder/URL/flag/code characters, path-ish charset only.
|
|
72
|
+
function checkablePath(s) {
|
|
73
|
+
if (/[<>*{}()$"'`=,;!?]|\s/.test(s)) return false; // placeholders, code, prose
|
|
74
|
+
if (/^(https?:|#|--|\/)/.test(s)) return false; // URLs, anchors, flags, site routes
|
|
75
|
+
// NB: `../` refs ARE checked — a space kernel points at `../augur/agents/*.md` (the
|
|
76
|
+
// engine sibling, present locally and cloned next to the checkout in CI), so a renamed
|
|
77
|
+
// engine doc must fail here. resolves() joins them against the .md's own dir.
|
|
78
|
+
if (!s.includes("/")) return false; // bare names: not checkable in general
|
|
79
|
+
if (/[:@]/.test(s)) return false; // scoped pkgs, remotes, key:value
|
|
80
|
+
return /^[\w.-]+(\/[\w.-]+)+\/?$/.test(s);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function resolves(ref, repo, mdDir) {
|
|
84
|
+
const clean = ref.replace(/\/$/, "");
|
|
85
|
+
// Only judge paths whose first segment is a real dir here — anything else is
|
|
86
|
+
// prose alternation ("upsert/del/rename") or a path rooted in another repo.
|
|
87
|
+
const seg0 = clean.split("/")[0];
|
|
88
|
+
if (!fs.existsSync(path.join(repo, seg0)) && !fs.existsSync(path.join(mdDir, seg0))) return true;
|
|
89
|
+
if (fs.existsSync(path.join(repo, clean)) || fs.existsSync(path.join(mdDir, clean))) return true;
|
|
90
|
+
// Gitignored paths (local-only by design, absent in a fresh clone) are not drift.
|
|
91
|
+
return [repo, mdDir].some(
|
|
92
|
+
(base) => spawnSync("git", ["check-ignore", "-q", path.join(base, clean)], { cwd: repo }).status === 0
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
for (const repo of repos) {
|
|
97
|
+
let scripts = {};
|
|
98
|
+
try { scripts = JSON.parse(fs.readFileSync(path.join(repo, "package.json"), "utf8")).scripts || {}; } catch {}
|
|
99
|
+
|
|
100
|
+
for (const file of mdFiles(repo)) {
|
|
101
|
+
const rel = path.relative(process.cwd(), file);
|
|
102
|
+
const mdDir = path.dirname(file);
|
|
103
|
+
const lines = fs.readFileSync(file, "utf8").split("\n");
|
|
104
|
+
let inFence = false;
|
|
105
|
+
|
|
106
|
+
lines.forEach((line, i) => {
|
|
107
|
+
const no = i + 1;
|
|
108
|
+
if (/^\s*(```|~~~)/.test(line)) { inFence = !inFence; return; }
|
|
109
|
+
|
|
110
|
+
// a. backtick-quoted file paths (inline code only — fenced blocks are examples)
|
|
111
|
+
if (!inFence) {
|
|
112
|
+
for (const m of line.matchAll(/`([^`]+)`/g)) {
|
|
113
|
+
const ref = m[1];
|
|
114
|
+
if (checkablePath(ref) && !resolves(ref, repo, mdDir)) fail(rel, no, "path", ref);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// b. npm run <script> — checked everywhere, fences included (recipes must work)
|
|
119
|
+
for (const m of line.matchAll(/npm (?:--prefix \S+ )?run (?:-s )?([\w:.-]+)/g)) {
|
|
120
|
+
if (/--prefix/.test(m[0]) && !m[0].includes(`--prefix ${path.basename(repo)} `)) continue; // other repo's scripts
|
|
121
|
+
if (!(m[1] in scripts)) fail(rel, no, "npm-script", m[1]);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// c. markdown links to relative files / anchors
|
|
125
|
+
if (inFence) return;
|
|
126
|
+
for (const m of line.matchAll(/\[[^\]]*\]\(([^)\s]+)\)/g)) {
|
|
127
|
+
const target = m[1];
|
|
128
|
+
if (/^(https?:|mailto:|\/)/.test(target)) continue;
|
|
129
|
+
const [fp, anchor] = target.split("#");
|
|
130
|
+
const dest = fp ? (fs.existsSync(path.join(mdDir, fp)) ? path.join(mdDir, fp)
|
|
131
|
+
: fs.existsSync(path.join(repo, fp)) ? path.join(repo, fp) : null)
|
|
132
|
+
: file; // pure #anchor → this file
|
|
133
|
+
if (!dest) { fail(rel, no, "link", target); continue; }
|
|
134
|
+
if (anchor && dest.endsWith(".md") && !headingsOf(dest).has(anchor.toLowerCase()))
|
|
135
|
+
fail(rel, no, "anchor", target);
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
for (const f of failures) console.log(f);
|
|
142
|
+
if (!failures.length && !quiet) console.log(`doc-lint: OK (${repos.length} repo${repos.length > 1 ? "s" : ""})`);
|
|
143
|
+
process.exit(failures.length ? 1 : 0);
|
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
// export.mjs — take an off-Cloudflare copy of the bundle store.
|
|
2
|
+
//
|
|
3
|
+
// augur export --out <dir> the live state of every space + engine chrome
|
|
4
|
+
// … --history every retained version's manifest and blobs too.
|
|
5
|
+
// ⚠️ A RESTORE DOES NOT REPLAY IT — see restore.mjs.
|
|
6
|
+
// This is an archive of what each past publish was,
|
|
7
|
+
// readable off disk; it does not rebuild the far
|
|
8
|
+
// side's rollback history.
|
|
9
|
+
// … --space <id> one space only
|
|
10
|
+
// … --full AND everything that is not published content:
|
|
11
|
+
// the roster, invites, publish tokens, statuses,
|
|
12
|
+
// card names, boards, comment threads, pins, and
|
|
13
|
+
// the images pasted onto a canvas
|
|
14
|
+
//
|
|
15
|
+
// Why this exists: the store is the only copy of what the site serves. Most of it
|
|
16
|
+
// is reproducible (clone the space at its recorded sha, publish again) — but a
|
|
17
|
+
// publish from a DIRTY working tree serves bytes that exist in no repository at
|
|
18
|
+
// all, and those are gone for good if the bucket is. R2 has no point-in-time
|
|
19
|
+
// restore to fall back on.
|
|
20
|
+
//
|
|
21
|
+
// Deliberately runs against the site's own HTTP API with a publish token rather
|
|
22
|
+
// than Cloudflare account credentials, so the nightly job that keeps the copy holds
|
|
23
|
+
// the weakest credential that can do the work — and so a restore needs nothing but
|
|
24
|
+
// this directory and a token.
|
|
25
|
+
//
|
|
26
|
+
// The output is incremental and content-addressed: blobs are named by hash and
|
|
27
|
+
// never rewritten, so re-running over an existing directory downloads only what is
|
|
28
|
+
// new. Point it at a path your normal backups already cover.
|
|
29
|
+
//
|
|
30
|
+
// ⚠️ WITHOUT `--full` THIS IS A COPY OF WHAT WAS PUBLISHED AND NOTHING ELSE. Not who
|
|
31
|
+
// could publish it, not who had been invited, not what anybody had said about it, not what
|
|
32
|
+
// had been pasted onto a board. That was the whole shape of a backup until
|
|
33
|
+
// `/__publish/_state/export` existed, and it is worth saying plainly rather than leaving
|
|
34
|
+
// somebody to discover it during a restore. `--full` needs a STAR-SCOPE token, because the
|
|
35
|
+
// answer carries the roster and the publish-token hashes.
|
|
36
|
+
//
|
|
37
|
+
// NOT included even with `--full`: config/instance.json (reproducible from the deploy
|
|
38
|
+
// shell, and not something to scatter extra copies of) and the password hashes, which the
|
|
39
|
+
// export endpoint cannot reach — a credential is account-level and belongs to the account
|
|
40
|
+
// store.
|
|
41
|
+
//
|
|
42
|
+
// SO HOW DOES ANYBODY SIGN IN TO A RESTORED INSTANCE? With the SEED password from the
|
|
43
|
+
// deploy shell's identity.json. `effectiveSecret` falls back to the roster's baked hash
|
|
44
|
+
// when the KV key is absent, and after a restore into a fresh instance it is absent — so
|
|
45
|
+
// the first admin gets in the same way they did on day one, and resets everybody from
|
|
46
|
+
// there. Nothing has to be carried and no KV surgery is needed. The nuance, because it is
|
|
47
|
+
// the difference between a promise kept and a promise nearly kept: that is the SEED
|
|
48
|
+
// password, not whatever they had changed it to. See test/restore-login.test.mjs.
|
|
49
|
+
//
|
|
50
|
+
// See `docs/2026-08-09-bundle-store-recovery.md`.
|
|
51
|
+
|
|
52
|
+
import { mkdir, writeFile, readFile, readdir, rename, unlink, stat } from "node:fs/promises";
|
|
53
|
+
import path from "node:path";
|
|
54
|
+
import { target, apiClient, buildStamp, idsFromStamp } from "./lib/store.mjs";
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Write a content-addressed file so that it is either absent or COMPLETE, never short.
|
|
58
|
+
*
|
|
59
|
+
* ⚠️ THIS IS WHAT MAKES THE RESUME SAFE, and without it the skip logic below is a lie.
|
|
60
|
+
* `writeFile` of a multi-megabyte buffer is several `write(2)` calls; a process killed
|
|
61
|
+
* between two of them leaves a file with the right NAME and the wrong bytes. The next run
|
|
62
|
+
* sees the name, skips it — "a hash that is present is by definition the right bytes" —
|
|
63
|
+
* and every run after that reports a complete copy. Nothing finds it until a restore, and
|
|
64
|
+
* a restore is exactly the moment nobody wants to find it.
|
|
65
|
+
*
|
|
66
|
+
* A rename within one directory is atomic on every filesystem this runs on, so the visible
|
|
67
|
+
* name never refers to a partial file. The `.part` is swept at startup, not left to rot.
|
|
68
|
+
*/
|
|
69
|
+
async function writeAtomic(file, buf) {
|
|
70
|
+
const tmp = `${file}.part`;
|
|
71
|
+
await writeFile(tmp, buf);
|
|
72
|
+
await rename(tmp, file);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Leftovers from a killed run: named so they can never be mistaken for a finished blob. */
|
|
76
|
+
async function sweepPartials(dir) {
|
|
77
|
+
let names = [];
|
|
78
|
+
try { names = await readdir(dir); } catch (e) { return 0; }
|
|
79
|
+
let n = 0;
|
|
80
|
+
for (const f of names) {
|
|
81
|
+
if (!f.endsWith(".part")) continue;
|
|
82
|
+
try { await unlink(path.join(dir, f)); n++; } catch (e) {}
|
|
83
|
+
}
|
|
84
|
+
return n;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const log = (msg) => console.error(`\x1b[36m[export]\x1b[0m ${msg}`);
|
|
88
|
+
const die = (msg) => { log(msg); process.exit(1); };
|
|
89
|
+
|
|
90
|
+
const args = process.argv.slice(2);
|
|
91
|
+
const flag = (f) => args.includes(f);
|
|
92
|
+
const opt = (f) => { const i = args.indexOf(f); return i >= 0 ? args[i + 1] : null; };
|
|
93
|
+
const OUT = opt("--out");
|
|
94
|
+
const HISTORY = flag("--history");
|
|
95
|
+
const FULL = flag("--full");
|
|
96
|
+
const ONE = opt("--space");
|
|
97
|
+
if (!OUT) die("name a destination: --out <dir>");
|
|
98
|
+
|
|
99
|
+
let origin, token;
|
|
100
|
+
try { ({ origin, token } = target()); } catch (e) { die(e.message); }
|
|
101
|
+
const req = apiClient(origin, token);
|
|
102
|
+
|
|
103
|
+
const started = Date.now();
|
|
104
|
+
// ⚠️ STAMPED BEFORE THE FIRST READ, never after the last one. `restore` refuses to bury
|
|
105
|
+
// live content published after this moment, so this has to be the moment the copy started
|
|
106
|
+
// describing the instance. Stamped at the END, a publish that landed DURING the export —
|
|
107
|
+
// after its manifest was read, before the copy finished — carries a `publishedAt` EARLIER
|
|
108
|
+
// than the copy's own, and the guard waves the restore through as if the copy were newer.
|
|
109
|
+
// The window is the whole length of the export, which on a real instance is minutes.
|
|
110
|
+
const exportedAt = new Date(started).toISOString();
|
|
111
|
+
log(`${origin} → ${OUT}${HISTORY ? " (with history)" : ""}${FULL ? " (with workspace state)" : ""}`);
|
|
112
|
+
|
|
113
|
+
const stamp = await buildStamp(origin);
|
|
114
|
+
const ids = (ONE ? [ONE] : idsFromStamp(stamp));
|
|
115
|
+
|
|
116
|
+
await mkdir(path.join(OUT, "manifests"), { recursive: true });
|
|
117
|
+
await mkdir(path.join(OUT, "blobs"), { recursive: true });
|
|
118
|
+
|
|
119
|
+
// Blobs already on disk from an earlier run, WITH THEIR SIZES. Content addressing makes
|
|
120
|
+
// the skip safe only for a file that is whole: a hash that is present is the right bytes
|
|
121
|
+
// as long as all of them are there, and the manifest records how many that is. So the
|
|
122
|
+
// resume compares the size it finds against the size the manifest declares and re-fetches
|
|
123
|
+
// anything short — which costs one `stat` per blob and closes the one way a resumed copy
|
|
124
|
+
// could report success over bytes a killed run left half-written.
|
|
125
|
+
const partials = await sweepPartials(path.join(OUT, "blobs"));
|
|
126
|
+
if (partials) log(`\x1b[33mswept ${partials} half-written blob(s) from an interrupted run\x1b[0m`);
|
|
127
|
+
const have = new Map();
|
|
128
|
+
try {
|
|
129
|
+
for (const f of await readdir(path.join(OUT, "blobs"))) {
|
|
130
|
+
if (f.endsWith(".part")) continue;
|
|
131
|
+
try { have.set(f, (await stat(path.join(OUT, "blobs", f))).size); } catch (e) {}
|
|
132
|
+
}
|
|
133
|
+
} catch (e) {}
|
|
134
|
+
if (have.size) log(`${have.size} blobs already present — downloading only what's new`);
|
|
135
|
+
|
|
136
|
+
// hash → { s: byte size, via: a space id whose manifest references it }.
|
|
137
|
+
// Blobs are global to the store (one content-addressed pool, shared across spaces),
|
|
138
|
+
// but the API path is scoped — /__publish/<space>/blob/<h> — and a per-space token
|
|
139
|
+
// is only accepted on its own space. So each hash remembers a space that references
|
|
140
|
+
// it, and is fetched through that one. A star-scoped token works either way.
|
|
141
|
+
const wanted = new Map();
|
|
142
|
+
const spaces = [];
|
|
143
|
+
const skipped = [];
|
|
144
|
+
|
|
145
|
+
for (const id of ids) {
|
|
146
|
+
let live;
|
|
147
|
+
try {
|
|
148
|
+
live = await (await req(`${id}/manifest`)).json();
|
|
149
|
+
} catch (e) {
|
|
150
|
+
// A space-scoped publish token can only read its own space, and backing up
|
|
151
|
+
// just your own space is a legitimate thing to do — so a 403 skips loudly
|
|
152
|
+
// instead of aborting. Everything else (a space in the stamp with no
|
|
153
|
+
// manifest, a typo in --space) is a real inconsistency: stop, rather than
|
|
154
|
+
// write a quietly incomplete copy.
|
|
155
|
+
if (/→ 403/.test(e.message)) {
|
|
156
|
+
log(`\x1b[33m⚠ ${id}: this token cannot read it — SKIPPED, not in this copy\x1b[0m`);
|
|
157
|
+
skipped.push({ id, reason: "forbidden" });
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
die(`${id}: could not read its manifest — ${e.message}`);
|
|
161
|
+
}
|
|
162
|
+
await writeFile(path.join(OUT, "manifests", `${id}.json`), JSON.stringify(live), "utf8");
|
|
163
|
+
for (const f of Object.values(live.files || {})) if (f && f.h && !wanted.has(f.h)) wanted.set(f.h, { s: f.s || 0, via: id });
|
|
164
|
+
spaces.push({ id, version: live.version || 0, publishedAt: live.publishedAt || null, source: live.source || null });
|
|
165
|
+
log(`${id}: v${live.version || 0}, ${Object.keys(live.files || {}).length} files${live.source && live.source.dirty ? " \x1b[33m[dirty]\x1b[0m" : ""}`);
|
|
166
|
+
|
|
167
|
+
if (!HISTORY) continue;
|
|
168
|
+
const { versions } = await (await req(`${id}/versions`)).json();
|
|
169
|
+
await mkdir(path.join(OUT, "versions", id), { recursive: true });
|
|
170
|
+
for (const v of versions) {
|
|
171
|
+
const m = await (await req(`${id}/version/${v}`)).json();
|
|
172
|
+
await writeFile(path.join(OUT, "versions", id, `${v}.json`), JSON.stringify(m), "utf8");
|
|
173
|
+
for (const f of Object.values(m.files || {})) if (f && f.h && !wanted.has(f.h)) wanted.set(f.h, { s: f.s || 0, via: id });
|
|
174
|
+
}
|
|
175
|
+
log(`${id}: ${versions.length} historical version(s)`);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// A blob counts as present only if it is present AND the right length. A recorded size of
|
|
179
|
+
// zero means the manifest does not say, and there is nothing to check it against.
|
|
180
|
+
const short = [];
|
|
181
|
+
const todo = [...wanted.keys()].filter((h) => {
|
|
182
|
+
if (!have.has(h)) return true;
|
|
183
|
+
const want = wanted.get(h).s;
|
|
184
|
+
if (want && have.get(h) !== want) { short.push(h); return true; }
|
|
185
|
+
return false;
|
|
186
|
+
});
|
|
187
|
+
if (short.length) {
|
|
188
|
+
log(`\x1b[33m⚠ ${short.length} blob(s) on disk are the wrong length — re-fetching. `
|
|
189
|
+
+ `An earlier copy in this directory was INCOMPLETE and reported success.\x1b[0m`);
|
|
190
|
+
}
|
|
191
|
+
const bytes = todo.reduce((n, h) => n + wanted.get(h).s, 0);
|
|
192
|
+
log(`${wanted.size} blobs referenced, ${todo.length} to fetch (${(bytes / 1e6).toFixed(1)} MB)`);
|
|
193
|
+
|
|
194
|
+
let done = 0, failed = 0;
|
|
195
|
+
const queue = [...todo];
|
|
196
|
+
await Promise.all(Array.from({ length: 8 }, async () => {
|
|
197
|
+
while (queue.length) {
|
|
198
|
+
const h = queue.pop();
|
|
199
|
+
for (let attempt = 0; ; attempt++) {
|
|
200
|
+
try {
|
|
201
|
+
const buf = Buffer.from(await (await req(`${wanted.get(h).via}/blob/${h}`)).arrayBuffer());
|
|
202
|
+
await writeAtomic(path.join(OUT, "blobs", h), buf);
|
|
203
|
+
done++;
|
|
204
|
+
if (done % 200 === 0) log(`${done}/${todo.length} blobs…`);
|
|
205
|
+
break;
|
|
206
|
+
} catch (e) {
|
|
207
|
+
if (attempt >= 2) { failed++; log(`blob ${h.slice(0, 12)} failed: ${e.message}`); break; }
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}));
|
|
212
|
+
|
|
213
|
+
// A partial copy that reports success is worse than no copy: it is the one you
|
|
214
|
+
// discover is short a few thousand blobs on the day you need it.
|
|
215
|
+
if (failed) die(`${failed} blob(s) failed — this copy is INCOMPLETE, do not trust it for restore.`);
|
|
216
|
+
|
|
217
|
+
// ── everything that is not published content ────────────────────────────────
|
|
218
|
+
// One document plus one file per canvas image. Written LAST, so a copy that has it has
|
|
219
|
+
// everything before it too — and recorded in export.json so a restore never has to guess
|
|
220
|
+
// whether it is looking at a full copy or a content-only one.
|
|
221
|
+
let state = null;
|
|
222
|
+
if (FULL) {
|
|
223
|
+
let doc;
|
|
224
|
+
try {
|
|
225
|
+
doc = await (await req("_state/export")).json();
|
|
226
|
+
} catch (e) {
|
|
227
|
+
die(/→ 403/.test(e.message)
|
|
228
|
+
? "--full needs a STAR-SCOPE token: it reads the roster, the invites and the publish-token hashes."
|
|
229
|
+
: `could not read the workspace state — ${e.message}`);
|
|
230
|
+
}
|
|
231
|
+
if (doc.failed && doc.failed.length) {
|
|
232
|
+
die(`the instance could not read ${doc.failed.length} state famil(y/ies): `
|
|
233
|
+
+ `${doc.failed.map((f) => f.id).join(", ")}. This copy would be INCOMPLETE — fix that first.`);
|
|
234
|
+
}
|
|
235
|
+
await writeFile(path.join(OUT, "state.json"), JSON.stringify(doc), "utf8");
|
|
236
|
+
|
|
237
|
+
// The canvas images the rows point at. Their bytes are in R2 under a different prefix
|
|
238
|
+
// from the published blobs, so the blob walk above never sees them.
|
|
239
|
+
await mkdir(path.join(OUT, "assets"), { recursive: true });
|
|
240
|
+
await sweepPartials(path.join(OUT, "assets"));
|
|
241
|
+
const already = new Set();
|
|
242
|
+
try { for (const f of await readdir(path.join(OUT, "assets"))) { if (!f.endsWith(".part")) already.add(f); } } catch (e) {}
|
|
243
|
+
|
|
244
|
+
// WHAT TYPE OF IMAGE EACH ONE IS, kept beside the bytes.
|
|
245
|
+
//
|
|
246
|
+
// The hash addresses the bytes and says nothing about how to serve them. The instance
|
|
247
|
+
// holds the type separately (R2 object metadata), the restore endpoint takes it from the
|
|
248
|
+
// request's `content-type` header, and a PUT that sends none is stored as `image/jpeg` —
|
|
249
|
+
// so without this every restored PNG, GIF and WebP arrives declared as a JPEG, cached
|
|
250
|
+
// `immutable` for a year, on the far side of a migration that reported every image
|
|
251
|
+
// present. It is a sidecar rather than a field on `state.json` because two of the three
|
|
252
|
+
// places an image can live carry no metadata row at all (see the export endpoint), and
|
|
253
|
+
// the response header is the one answer that exists for all three.
|
|
254
|
+
let types = {};
|
|
255
|
+
try { types = JSON.parse(await readFile(path.join(OUT, "assets.json"), "utf8")); } catch (e) {}
|
|
256
|
+
const hashes = (doc.assets || []).filter((h) => !already.has(h));
|
|
257
|
+
let assetFail = 0;
|
|
258
|
+
for (const h of hashes) {
|
|
259
|
+
try {
|
|
260
|
+
const res = await req(`_state/asset/${h}`);
|
|
261
|
+
const ct = (res.headers.get("content-type") || "").split(";")[0].trim();
|
|
262
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
263
|
+
await writeAtomic(path.join(OUT, "assets", h), buf);
|
|
264
|
+
if (ct) types[h] = ct;
|
|
265
|
+
} catch (e) { assetFail++; log(`asset ${h.slice(0, 12)} failed: ${e.message}`); }
|
|
266
|
+
}
|
|
267
|
+
if (assetFail) die(`${assetFail} canvas image(s) failed — this copy is INCOMPLETE, do not trust it for restore.`);
|
|
268
|
+
await writeFile(path.join(OUT, "assets.json"), JSON.stringify(types), "utf8");
|
|
269
|
+
state = {
|
|
270
|
+
families: Object.keys(doc.families || {}).length,
|
|
271
|
+
absent: (doc.absent || []).length,
|
|
272
|
+
assets: (doc.assets || []).length,
|
|
273
|
+
};
|
|
274
|
+
log(`workspace state: ${state.families} famil(y/ies), ${state.assets} canvas image(s)`);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
await writeFile(path.join(OUT, "export.json"), JSON.stringify({
|
|
278
|
+
format: 1,
|
|
279
|
+
origin,
|
|
280
|
+
exportedAt,
|
|
281
|
+
finishedAt: new Date().toISOString(),
|
|
282
|
+
history: HISTORY,
|
|
283
|
+
// `full: false` is written, not omitted: a restore that could not tell a content-only
|
|
284
|
+
// copy from a full one would report "no state to replay" for both.
|
|
285
|
+
full: FULL,
|
|
286
|
+
state,
|
|
287
|
+
spaces,
|
|
288
|
+
// What this copy does NOT contain, recorded so a restore from it is never a
|
|
289
|
+
// surprise. Engine chrome landing here is benign — it rebuilds from the engine
|
|
290
|
+
// repo with one deploy — but it should still be stated, not inferred.
|
|
291
|
+
skipped,
|
|
292
|
+
blobs: wanted.size,
|
|
293
|
+
}, null, 2), "utf8");
|
|
294
|
+
|
|
295
|
+
log(`done in ${((Date.now() - started) / 1000).toFixed(1)}s`);
|
|
296
|
+
if (skipped.length) log(`\x1b[33m${skipped.length} target(s) skipped: ${skipped.map((s) => s.id).join(", ")}\x1b[0m`);
|
|
297
|
+
console.log(`${OUT} ${spaces.length} space(s), ${wanted.size} blobs`
|
|
298
|
+
+ (state ? `, ${state.families} state famil(y/ies), ${state.assets} image(s)` : "")
|
|
299
|
+
+ (skipped.length ? `, ${skipped.length} skipped` : ""));
|
|
300
|
+
if (!FULL) log("\x1b[33mcontent only — pass --full to include the roster, comments, boards and pins\x1b[0m");
|