@yojahny/wp-design-library 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/package.json +1 -1
- package/src/artifacts.mjs +9 -1
- package/src/cli/add.mjs +30 -18
- package/src/cli/check.mjs +45 -0
- package/src/cli/index.mjs +1 -1
- package/src/cli/refresh.mjs +4 -3
- package/src/cli/serve.mjs +3 -2
- package/src/cli/ui.mjs +1 -1
- package/src/entry.mjs +9 -0
- package/src/index/build.mjs +51 -9
- package/src/index/embed.mjs +7 -2
- package/src/index/veccache.mjs +53 -0
- package/src/ingest/fields.mjs +77 -0
- package/src/mcp/http.mjs +16 -1
- package/src/mcp/prompts.mjs +8 -2
- package/src/mcp/tools.mjs +41 -0
- package/src/r2.mjs +67 -12
- package/src/ui/app.js +5 -1
- package/src/ui/build.mjs +53 -15
- package/src/ui/templates/index.html +24 -1
package/README.md
CHANGED
|
@@ -172,6 +172,15 @@ blanks. The same works from the plugin: with the MCP server registered, say
|
|
|
172
172
|
"add `~/Desktop/site.webp` to the design library as `acme-dark-hero`" and the
|
|
173
173
|
session calls `add`, reads the strip, fills the draft, and calls `save_entry`.
|
|
174
174
|
|
|
175
|
+
Ingest needs no filesystem: `add` → `set_fields` → `save_entry` closes entirely over
|
|
176
|
+
MCP, so it works against a hosted instance as well as a local checkout. `set_fields`
|
|
177
|
+
writes judgement fields only — roles, feel, motion, palette, type, body, tier — and
|
|
178
|
+
refuses provenance keys by name, so `captured.*` and the artifact digests stay the
|
|
179
|
+
server's record of what it measured. `get_vocab` returns the controlled terms, and
|
|
180
|
+
`add_batch` drafts every file in the server's inbox in one run, resuming where an
|
|
181
|
+
interrupted run stopped. Editing `entries/<slug>/entry.md` by hand remains available in
|
|
182
|
+
a local checkout and does the same thing.
|
|
183
|
+
|
|
175
184
|
What goes where, with the values checked against `vocab.yaml`:
|
|
176
185
|
|
|
177
186
|
| Field | What to write |
|
|
@@ -248,6 +257,7 @@ The plugin does this for you, but from the CLI:
|
|
|
248
257
|
```bash
|
|
249
258
|
node bin/library.mjs index # rebuild the index from entries/
|
|
250
259
|
node bin/library.mjs check # validate every entry, 1 line per problem
|
|
260
|
+
node bin/library.mjs check --remote # + confirm every recorded artifact key resolves in R2
|
|
251
261
|
```
|
|
252
262
|
|
|
253
263
|
Over MCP, seven tools: `search` (keyword + facet filters, aliases resolved),
|
package/package.json
CHANGED
package/src/artifacts.mjs
CHANGED
|
@@ -17,7 +17,15 @@ export function cachePath(p, key) {
|
|
|
17
17
|
// migrate-artifacts.mjs documents at length; the fix had not reached this file.
|
|
18
18
|
export async function artifactFile(p, artifact, { config, fetcher = fetch } = {}) {
|
|
19
19
|
const file = cachePath(p, artifact.key);
|
|
20
|
-
|
|
20
|
+
// bytes is the cheap first gate -- it short-circuits the common mismatch without hashing. A
|
|
21
|
+
// length match is not proof: bytes can rewrite in place at the same length, so a HIT is now
|
|
22
|
+
// verified against the digest too. A hit that fails is a corrupt cache entry, not a dead
|
|
23
|
+
// reader -- self-heal by deleting it and falling through to the fetch below, the same path a
|
|
24
|
+
// genuine miss takes.
|
|
25
|
+
if (fs.existsSync(file) && fs.statSync(file).size === artifact.bytes) {
|
|
26
|
+
if (sha256(fs.readFileSync(file)) === artifact.sha256) return file;
|
|
27
|
+
fs.rmSync(file);
|
|
28
|
+
}
|
|
21
29
|
config ??= readConfig();
|
|
22
30
|
const url = objectUrl(config, artifact.key);
|
|
23
31
|
const response = await fetcher(url, { signal: AbortSignal.timeout(30000) });
|
package/src/cli/add.mjs
CHANGED
|
@@ -8,7 +8,7 @@ import { attachCapture } from '../ingest/attach.mjs';
|
|
|
8
8
|
import { measure } from '../ingest/measure.mjs';
|
|
9
9
|
import { loadVocab, resolveTerm } from '../vocab.mjs';
|
|
10
10
|
import { parseEntry } from '../entry.mjs';
|
|
11
|
-
import { artifactKey, putAndVerify, r2Config, s3Upload } from '../r2.mjs';
|
|
11
|
+
import { artifactKey, putAndVerify, r2Config, s3Head, s3Upload } from '../r2.mjs';
|
|
12
12
|
|
|
13
13
|
function opt(args, name) { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; }
|
|
14
14
|
|
|
@@ -43,7 +43,7 @@ async function stripFromFrames(framePaths, out) {
|
|
|
43
43
|
// URL/file:// input: measure the live page with system Chrome instead of sampling
|
|
44
44
|
// frames from a recording. Scratch-dir-first, same as the frame path: nothing under
|
|
45
45
|
// entries/ is created OR destroyed until every byte is captured AND uploaded.
|
|
46
|
-
async function addFromUrl({ p, input, slug, title, source, dir, record, force, attach, config, upload, fetcher }) {
|
|
46
|
+
async function addFromUrl({ p, input, slug, title, source, dir, record, force, attach, config, upload, head, fetcher }) {
|
|
47
47
|
const fs = await import('node:fs');
|
|
48
48
|
const os = await import('node:os');
|
|
49
49
|
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'wpdl-add-'));
|
|
@@ -88,8 +88,8 @@ async function addFromUrl({ p, input, slug, title, source, dir, record, force, a
|
|
|
88
88
|
// network round trip, and it throws for reasons this process does not control (absent
|
|
89
89
|
// credentials, a reset connection, a digest mismatch). Uploading after the wipe made every one
|
|
90
90
|
// of those a data-loss bug: `--force` deleted the entry and put nothing in its place.
|
|
91
|
-
// config/upload/fetcher are resolved lazily, here, and not as addEntry's defaults, so a
|
|
92
|
-
// earlier in the flow never needs R2 credentials.
|
|
91
|
+
// config/upload/head/fetcher are resolved lazily, here, and not as addEntry's defaults, so a
|
|
92
|
+
// refusal earlier in the flow never needs R2 credentials.
|
|
93
93
|
const names = ['strip.webp'];
|
|
94
94
|
if (result.ok) names.push('measure.json', 'screen-1440.png', 'screen-390.png');
|
|
95
95
|
const types = { '.webp': 'image/webp', '.png': 'image/png', '.json': 'application/json' };
|
|
@@ -97,10 +97,11 @@ async function addFromUrl({ p, input, slug, title, source, dir, record, force, a
|
|
|
97
97
|
try {
|
|
98
98
|
config ??= r2Config();
|
|
99
99
|
if (!upload) upload = await s3Upload(config);
|
|
100
|
+
if (!head) head = await s3Head(config);
|
|
100
101
|
for (const name of names) {
|
|
101
102
|
const body = fs.readFileSync(path.join(scratch, name));
|
|
102
103
|
const key = artifactKey(slug, index, name);
|
|
103
|
-
artifacts.push(await putAndVerify({ config, key, body, contentType: types[path.extname(name)], upload, fetcher }));
|
|
104
|
+
artifacts.push(await putAndVerify({ config, key, body, contentType: types[path.extname(name)], upload, head, fetcher }));
|
|
104
105
|
}
|
|
105
106
|
} catch (err) {
|
|
106
107
|
fs.rmSync(scratch, { recursive: true, force: true });
|
|
@@ -131,7 +132,7 @@ async function addFromUrl({ p, input, slug, title, source, dir, record, force, a
|
|
|
131
132
|
return { dir: d.dir, strip, blanks: d.blanks, measured: result.ok };
|
|
132
133
|
}
|
|
133
134
|
|
|
134
|
-
export async function addEntry({ input, slug, title, source, force = false, frames, record = false, attach = false, config, upload, fetcher = fetch }) {
|
|
135
|
+
export async function addEntry({ input, slug, title, source, force = false, frames, record = false, attach = false, config, upload, head, fetcher = fetch }) {
|
|
135
136
|
const p = paths();
|
|
136
137
|
if (!input || !slug || !title || !source?.kind || !source?.license) throw new Error('add: input, slug, title, --source and --license are required');
|
|
137
138
|
// Refused here (draft time), not first at save: resolveTerm is the single vocab gate
|
|
@@ -147,7 +148,7 @@ export async function addEntry({ input, slug, title, source, force = false, fram
|
|
|
147
148
|
return { refused: `entry ${slug} exists; pass attach to add a capture, or force to overwrite` };
|
|
148
149
|
}
|
|
149
150
|
if (URL_RE.test(input)) {
|
|
150
|
-
return addFromUrl({ p, input, slug, title, source, dir, record, force, attach, config, upload, fetcher });
|
|
151
|
+
return addFromUrl({ p, input, slug, title, source, dir, record, force, attach, config, upload, head, fetcher });
|
|
151
152
|
}
|
|
152
153
|
// Sample into a scratch dir first: a refusal must leave nothing under entries/.
|
|
153
154
|
const scratch = mkdtempSync(path.join(os.tmpdir(), 'wpdl-add-'));
|
|
@@ -170,9 +171,10 @@ export async function addEntry({ input, slug, title, source, force = false, fram
|
|
|
170
171
|
try {
|
|
171
172
|
config ??= r2Config();
|
|
172
173
|
if (!upload) upload = await s3Upload(config);
|
|
174
|
+
if (!head) head = await s3Head(config);
|
|
173
175
|
stripArtifact = await putAndVerify({
|
|
174
176
|
config, key: artifactKey(slug, index, 'strip.webp'), body: readFileSync(scratchStrip),
|
|
175
|
-
contentType: 'image/webp', upload, fetcher,
|
|
177
|
+
contentType: 'image/webp', upload, head, fetcher,
|
|
176
178
|
});
|
|
177
179
|
} catch (err) {
|
|
178
180
|
rmSync(scratch, { recursive: true, force: true });
|
|
@@ -214,30 +216,40 @@ export async function addEntry({ input, slug, title, source, force = false, fram
|
|
|
214
216
|
return { dir: d.dir, strip, blanks: d.blanks, frames: media.frames };
|
|
215
217
|
}
|
|
216
218
|
|
|
217
|
-
export async function addInbox({ source, config, upload, fetcher }) {
|
|
219
|
+
export async function addInbox({ source, config, upload, head, fetcher }) {
|
|
218
220
|
const p = paths();
|
|
219
221
|
const fs = await import('node:fs');
|
|
220
222
|
const inbox = path.join(p.data, 'inbox');
|
|
221
223
|
const out = [];
|
|
222
224
|
if (!fs.existsSync(inbox)) return out;
|
|
225
|
+
// One file per run, not one shared log: a settled 2,000-file inbox re-run appends 2,000 more
|
|
226
|
+
// `skipped` lines to a file that never shrinks, every time. The ISO stamp also sorts by name,
|
|
227
|
+
// so "what did the run that died last night do" is answerable by reading the newest file.
|
|
228
|
+
const runLog = path.join(p.data, '.library', 'runs', `inbox-${new Date().toISOString().replace(/[:.]/g, '-')}.jsonl`);
|
|
229
|
+
fs.mkdirSync(path.dirname(runLog), { recursive: true });
|
|
230
|
+
// Appended as each file finishes, not written at the end: a run killed at entry 1,400 never
|
|
231
|
+
// reaches its own return statement, and that is the case this exists for. One complete line per
|
|
232
|
+
// append, so a process killed mid-write loses at most its last record rather than corrupting the
|
|
233
|
+
// file. .library/ is already gitignored.
|
|
234
|
+
const record = (r) => { fs.appendFileSync(runLog, JSON.stringify({ ...r, at: new Date().toISOString() }) + '\n'); return r; };
|
|
223
235
|
const seenSlugs = new Map(); // ponytail: slug -> first file this run, names the collision
|
|
224
236
|
for (const file of fs.readdirSync(inbox).filter((f) => MEDIA.test(f)).sort()) {
|
|
225
237
|
const slug = slugify(file);
|
|
226
|
-
if (!slug) { out.push({ file, slug, status: 'refused', reason: 'cannot derive a slug from the file name; pass --slug' }); continue; }
|
|
227
|
-
if (seenSlugs.has(slug)) { out.push({ file, slug, status: 'refused', reason: `slug ${slug} collides with ${seenSlugs.get(slug)} in this run` }); continue; }
|
|
238
|
+
if (!slug) { out.push(record({ file, slug, status: 'refused', reason: 'cannot derive a slug from the file name; pass --slug' })); continue; }
|
|
239
|
+
if (seenSlugs.has(slug)) { out.push(record({ file, slug, status: 'refused', reason: `slug ${slug} collides with ${seenSlugs.get(slug)} in this run` })); continue; }
|
|
228
240
|
seenSlugs.set(slug, file);
|
|
229
241
|
const full = path.join(inbox, file);
|
|
230
|
-
if (fs.existsSync(path.join(p.entries, slug, 'entry.md'))) { out.push({ file, slug, status: 'skipped', reason: 'entry exists' }); continue; }
|
|
242
|
+
if (fs.existsSync(path.join(p.entries, slug, 'entry.md'))) { out.push(record({ file, slug, status: 'skipped', reason: 'entry exists' })); continue; }
|
|
231
243
|
let meta = {};
|
|
232
244
|
const side = full + '.json';
|
|
233
|
-
if (fs.existsSync(side)) { try { meta = JSON.parse(fs.readFileSync(side, 'utf8')); } catch (e) { out.push({ file, slug, status: 'refused', reason: 'sidecar is not JSON' }); continue; } }
|
|
234
|
-
if (meta === null || typeof meta !== 'object' || Array.isArray(meta)) { out.push({ file, slug, status: 'refused', reason: 'sidecar must be a JSON object' }); continue; }
|
|
245
|
+
if (fs.existsSync(side)) { try { meta = JSON.parse(fs.readFileSync(side, 'utf8')); } catch (e) { out.push(record({ file, slug, status: 'refused', reason: 'sidecar is not JSON' })); continue; } }
|
|
246
|
+
if (meta === null || typeof meta !== 'object' || Array.isArray(meta)) { out.push(record({ file, slug, status: 'refused', reason: 'sidecar must be a JSON object' })); continue; }
|
|
235
247
|
const kind = meta.kind ?? source?.kind, license = meta.license ?? source?.license;
|
|
236
|
-
if (!kind || !license) { out.push({ file, slug, status: 'refused', reason: 'no source kind/license: add a <file>.json sidecar or pass --source and --license' }); continue; }
|
|
248
|
+
if (!kind || !license) { out.push(record({ file, slug, status: 'refused', reason: 'no source kind/license: add a <file>.json sidecar or pass --source and --license' })); continue; }
|
|
237
249
|
try {
|
|
238
|
-
const r = await addEntry({ input: full, slug, title: meta.title ?? slug.replace(/-/g, ' '), source: { kind, url: meta.url ?? source?.url, license }, config, upload, fetcher });
|
|
239
|
-
out.push(r.refused ? { file, slug, status: 'refused', reason: r.refused } : { file, slug, status: 'drafted', frames: r.frames });
|
|
240
|
-
} catch (e) { out.push({ file, slug, status: 'refused', reason: e.message }); }
|
|
250
|
+
const r = await addEntry({ input: full, slug, title: meta.title ?? slug.replace(/-/g, ' '), source: { kind, url: meta.url ?? source?.url, license }, config, upload, head, fetcher });
|
|
251
|
+
out.push(record(r.refused ? { file, slug, status: 'refused', reason: r.refused } : { file, slug, status: 'drafted', frames: r.frames }));
|
|
252
|
+
} catch (e) { out.push(record({ file, slug, status: 'refused', reason: e.message })); }
|
|
241
253
|
}
|
|
242
254
|
return out;
|
|
243
255
|
}
|
package/src/cli/check.mjs
CHANGED
|
@@ -4,6 +4,17 @@ import { loadVocab } from '../vocab.mjs';
|
|
|
4
4
|
import { listEntries, parseEntry, validateEntry } from '../entry.mjs';
|
|
5
5
|
import { fetchModels, modelDirs } from '../index/embed.mjs';
|
|
6
6
|
|
|
7
|
+
// A fixed-width worker pool, not one request at a time: ~2000 entries at up to four artifacts
|
|
8
|
+
// each is ~8000 HEADs, and sequential that is minutes at a realistic edge RTT and tens of hours
|
|
9
|
+
// against a bucket that black-holes connections (each request burning the full timeout below). 8
|
|
10
|
+
// is a guess, not tuned -- enough to matter, low enough that a real outage's failures still read
|
|
11
|
+
// as a report instead of a wall of interleaved errors.
|
|
12
|
+
async function mapPool(items, limit, fn) {
|
|
13
|
+
let i = 0;
|
|
14
|
+
const worker = async () => { while (i < items.length) { const idx = i++; await fn(items[idx], idx); } };
|
|
15
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
16
|
+
}
|
|
17
|
+
|
|
7
18
|
export async function run(args = []) {
|
|
8
19
|
const p = paths();
|
|
9
20
|
if (args.includes('--fetch-models')) {
|
|
@@ -15,13 +26,47 @@ export async function run(args = []) {
|
|
|
15
26
|
}
|
|
16
27
|
const vocab = loadVocab(p.vocab);
|
|
17
28
|
let bad = 0, n = 0;
|
|
29
|
+
const artifacts = [];
|
|
18
30
|
for (const dir of listEntries(p.entries)) {
|
|
19
31
|
n++;
|
|
20
32
|
let e;
|
|
21
33
|
try { e = parseEntry(dir); } catch (err) { bad++; process.stderr.write(`${path.basename(dir)}: ${err.message}\n`); continue; }
|
|
22
34
|
const r = validateEntry(e, vocab);
|
|
23
35
|
if (!r.ok) { bad++; for (const m of r.errors) process.stderr.write(`${e.slug}: ${m}\n`); }
|
|
36
|
+
for (const c of e.fm.captured?.captures ?? []) for (const a of c.artifacts ?? []) artifacts.push(a);
|
|
24
37
|
}
|
|
25
38
|
process.stdout.write(`${n - bad}/${n} entries ok\n`);
|
|
39
|
+
|
|
40
|
+
// --remote answers the question `check` otherwise never asks: does every key this corpus
|
|
41
|
+
// CLAIMS is in R2 actually resolve. Metadata validation above never touches the network --
|
|
42
|
+
// that stays the default, so CI with no credentials keeps working -- this is opt-in.
|
|
43
|
+
if (args.includes('--remote')) {
|
|
44
|
+
const { readConfig, objectUrl } = await import('../r2.mjs');
|
|
45
|
+
// readConfig throws when R2_PUBLIC_BASE_URL is absent, and unguarded that reaches the operator
|
|
46
|
+
// as a raw Node stack trace -- the same "work it out from the wreckage" failure 1.0.1 fixed for
|
|
47
|
+
// the gallery. It needs only that one variable and no credentials, so the refusal can say
|
|
48
|
+
// exactly what to set.
|
|
49
|
+
let config;
|
|
50
|
+
try { config = readConfig(); }
|
|
51
|
+
catch (err) { process.stderr.write(`check --remote: ${err.message}\n`); return 1; }
|
|
52
|
+
// Collected and sorted before printing, not written as each HEAD lands -- 8 in flight would
|
|
53
|
+
// otherwise interleave and print in whatever order they happen to settle, so the same broken
|
|
54
|
+
// corpus would read differently run to run.
|
|
55
|
+
const problems = [];
|
|
56
|
+
await mapPool(artifacts, 8, async (a) => {
|
|
57
|
+
try {
|
|
58
|
+
const r = await fetch(objectUrl(config, a.key), { method: 'HEAD', signal: AbortSignal.timeout(20000) });
|
|
59
|
+
// Existence AND size. A 200 with the wrong length means the object is not what the entry
|
|
60
|
+
// records -- exactly as broken as it being absent, and cheaper to catch here than at read
|
|
61
|
+
// time on a visitor's page.
|
|
62
|
+
if (!r.ok) { problems.push(`${a.key}: HTTP ${r.status}`); return; }
|
|
63
|
+
const len = Number(r.headers.get('content-length'));
|
|
64
|
+
if (len !== a.bytes) problems.push(`${a.key}: expected ${a.bytes} bytes, bucket says ${len}`);
|
|
65
|
+
} catch (err) { problems.push(`${a.key}: ${err.message}`); }
|
|
66
|
+
});
|
|
67
|
+
for (const p of problems.sort()) process.stderr.write(`${p}\n`);
|
|
68
|
+
process.stdout.write(`${artifacts.length - problems.length}/${artifacts.length} artifacts reachable\n`);
|
|
69
|
+
if (problems.length) return 1;
|
|
70
|
+
}
|
|
26
71
|
return bad ? 1 : 0;
|
|
27
72
|
}
|
package/src/cli/index.mjs
CHANGED
|
@@ -6,7 +6,7 @@ export async function run() {
|
|
|
6
6
|
const p = paths();
|
|
7
7
|
const r = await buildIndex({ entriesDir: p.entries, indexPath: p.index, vocab: loadVocab(p.vocab), models: p.models, p });
|
|
8
8
|
for (const m of r.refused) process.stderr.write(`refused ${m}\n`);
|
|
9
|
-
process.stdout.write(`indexed ${r.entries} entries, refused: ${r.refused.length}, arms: ${r.arms.join(',')}\n`);
|
|
9
|
+
process.stdout.write(`indexed ${r.entries} entries, refused: ${r.refused.length}, arms: ${r.arms.join(',')}, embedded ${r.embedded.text} text and ${r.embedded.image} image vectors\n`);
|
|
10
10
|
for (const s of r.skipped) process.stderr.write(`embeddings: skipped (${s.reason})\n`);
|
|
11
11
|
return 0;
|
|
12
12
|
}
|
package/src/cli/refresh.mjs
CHANGED
|
@@ -8,7 +8,7 @@ import { loadVocab } from '../vocab.mjs';
|
|
|
8
8
|
import { buildIndex } from '../index/build.mjs';
|
|
9
9
|
import { measure } from '../ingest/measure.mjs';
|
|
10
10
|
import { refuseUnsafeUrl } from '../ingest/url-guard.mjs';
|
|
11
|
-
import { artifactKey, putAndVerify, r2Config, s3Upload } from '../r2.mjs';
|
|
11
|
+
import { artifactKey, putAndVerify, r2Config, s3Head, s3Upload } from '../r2.mjs';
|
|
12
12
|
|
|
13
13
|
const REFRESHED = ['measure.json', 'screen-1440.png', 'screen-390.png'];
|
|
14
14
|
const TYPES = { '.png': 'image/png', '.json': 'application/json' };
|
|
@@ -19,7 +19,7 @@ const TYPES = { '.png': 'image/png', '.json': 'application/json' };
|
|
|
19
19
|
// the measurement and captured.at change. `transport` is undefined for the CLI, 'http'
|
|
20
20
|
// for the hosted MCP server; the SSRF guard only consults DNS in the latter case.
|
|
21
21
|
// `measurer` exists so this path can be driven without a browser; nothing but a test passes it.
|
|
22
|
-
export async function refreshEntry({ slug, transport, config, upload, fetcher = fetch, measurer = measure }) {
|
|
22
|
+
export async function refreshEntry({ slug, transport, config, upload, head, fetcher = fetch, measurer = measure }) {
|
|
23
23
|
const p = paths();
|
|
24
24
|
const dir = path.join(p.entries, slug);
|
|
25
25
|
let e;
|
|
@@ -61,10 +61,11 @@ export async function refreshEntry({ slug, transport, config, upload, fetcher =
|
|
|
61
61
|
try {
|
|
62
62
|
config ??= r2Config();
|
|
63
63
|
if (!upload) upload = await s3Upload(config);
|
|
64
|
+
if (!head) head = await s3Head(config);
|
|
64
65
|
uploaded = [];
|
|
65
66
|
for (const f of REFRESHED) {
|
|
66
67
|
uploaded.push(await putAndVerify({ config, key: artifactKey(slug, 1, f), body: fs.readFileSync(path.join(scratch, f)),
|
|
67
|
-
contentType: TYPES[path.extname(f)], upload, fetcher }));
|
|
68
|
+
contentType: TYPES[path.extname(f)], upload, head, fetcher }));
|
|
68
69
|
}
|
|
69
70
|
} catch (err) {
|
|
70
71
|
fs.rmSync(scratch, { recursive: true, force: true });
|
package/src/cli/serve.mjs
CHANGED
|
@@ -66,7 +66,7 @@ async function buildTheIndex(p) {
|
|
|
66
66
|
const { loadVocab } = await import('../vocab.mjs');
|
|
67
67
|
const { buildIndex } = await import('../index/build.mjs');
|
|
68
68
|
const r = await buildIndex({ entriesDir: p.entries, indexPath: p.index, vocab: loadVocab(p.vocab), models: p.models, p });
|
|
69
|
-
process.stderr.write(`index: ${r.entries} entries, refused ${r.refused.length}\n`);
|
|
69
|
+
process.stderr.write(`index: ${r.entries} entries, refused ${r.refused.length}, arms: ${r.arms.join(',')}, embedded ${r.embedded.text} text and ${r.embedded.image} image vectors\n`);
|
|
70
70
|
for (const s of r.skipped) process.stderr.write(`embeddings: skipped (${s.reason})\n`);
|
|
71
71
|
}
|
|
72
72
|
|
|
@@ -93,7 +93,8 @@ export async function run(args) {
|
|
|
93
93
|
await buildTheIndex(p);
|
|
94
94
|
try {
|
|
95
95
|
const { buildUi } = await import('../ui/build.mjs');
|
|
96
|
-
await buildUi({ indexPath: p.index, entriesDir: p.entries, outDir: path.join(p.data, 'ui'), p });
|
|
96
|
+
const r = await buildUi({ indexPath: p.index, entriesDir: p.entries, outDir: path.join(p.data, 'ui'), p });
|
|
97
|
+
process.stderr.write(`ui: ${r.entries} entries, ${r.drafts} drafts, rendered ${r.rendered} thumbnails\n`);
|
|
97
98
|
} catch (err) {
|
|
98
99
|
// the gallery is a convenience for a human; the MCP API must not depend on it
|
|
99
100
|
process.stderr.write(`ui: build failed, serving without it: ${err.stack || err}\n`);
|
package/src/cli/ui.mjs
CHANGED
|
@@ -9,7 +9,7 @@ export async function run(args = []) {
|
|
|
9
9
|
const outDir = path.join(p.data, 'ui');
|
|
10
10
|
const previewMediaDir = args.includes('--local-media') ? path.join(p.data, '.library', 'media') : undefined;
|
|
11
11
|
const r = await buildUi({ indexPath: p.index, entriesDir: p.entries, outDir, previewMediaDir, p });
|
|
12
|
-
process.stdout.write(`ui: ${r.entries} entries, ${r.drafts} drafts, built into ${outDir}\n`);
|
|
12
|
+
process.stdout.write(`ui: ${r.entries} entries, ${r.drafts} drafts, built into ${outDir}, rendered ${r.rendered} thumbnails\n`);
|
|
13
13
|
if (args.includes('--serve')) {
|
|
14
14
|
const port = Number(opt(args, '--port') ?? 4180);
|
|
15
15
|
const { serveStatic } = await import('../ui/serve.mjs');
|
package/src/entry.mjs
CHANGED
|
@@ -19,6 +19,15 @@ export function parseEntry(dir) {
|
|
|
19
19
|
return { slug: fm.slug, dir, fm, body: m[2].trim() };
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
// The symmetric partner to parseEntry. Four other call sites serialize an entry
|
|
23
|
+
// inline with their own lineWidth; they are deliberately left alone, because
|
|
24
|
+
// changing a writer reflows YAML in files it was not asked to touch. New code
|
|
25
|
+
// uses this one.
|
|
26
|
+
export function writeEntry(dir, fm, body) {
|
|
27
|
+
const text = `---\n${yaml.dump(fm, { lineWidth: 120 })}---\n${body ? `\n${body}\n` : ''}`;
|
|
28
|
+
fs.writeFileSync(path.join(dir, 'entry.md'), text);
|
|
29
|
+
}
|
|
30
|
+
|
|
22
31
|
function get(obj, dotted) {
|
|
23
32
|
return dotted.split('.').reduce((o, k) => (o == null ? undefined : o[k]), obj);
|
|
24
33
|
}
|
package/src/index/build.mjs
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { createHash } from 'node:crypto';
|
|
4
5
|
import { listEntries, parseEntry, validateEntry } from '../entry.mjs';
|
|
5
6
|
import { openIndex } from './query.mjs';
|
|
6
7
|
import { getEmbedder, absentReason } from './embed.mjs';
|
|
7
8
|
import { artifactFile } from '../artifacts.mjs';
|
|
9
|
+
import { openVecCache } from './veccache.mjs';
|
|
8
10
|
|
|
9
11
|
const DIR = path.dirname(fileURLToPath(import.meta.url));
|
|
10
12
|
const SCHEMA = fs.readFileSync(path.join(DIR, 'schema.sql'), 'utf8');
|
|
@@ -26,11 +28,23 @@ export async function buildIndex({ entriesDir, indexPath, vocab, models, p }) {
|
|
|
26
28
|
// a killed build can leave a half-written .tmp behind; clear it so the open below always starts fresh
|
|
27
29
|
fs.rmSync(tmpPath, { force: true });
|
|
28
30
|
let db;
|
|
31
|
+
let cache = null;
|
|
29
32
|
try {
|
|
30
33
|
db = openIndex(tmpPath, { readonly: false });
|
|
31
34
|
db.exec(SCHEMA);
|
|
32
35
|
if (db.vec) db.exec(SCHEMA_VEC);
|
|
33
36
|
const emb = await getEmbedder({ models });
|
|
37
|
+
// The cache is only worth opening when there is an embedder and a vec arm to feed.
|
|
38
|
+
// .library/vectors.sqlite is derived, gitignored and regenerable: a reader that finds it
|
|
39
|
+
// broken (first-open lock contention on a fresh volume, an unopenable file) should recompute,
|
|
40
|
+
// not die -- the same ruling artifactFile already applies to a bad cache entry.
|
|
41
|
+
try { cache = emb && db.vec ? openVecCache(p) : null; }
|
|
42
|
+
catch (err) { process.stderr.write(`index: vector cache unavailable, recomputing: ${err.message}\n`); cache = null; }
|
|
43
|
+
// skipped.image is nested here, not returned as a sibling top-level `skipped`, because that
|
|
44
|
+
// name is already taken by the arm-level skip-reason array below (`skipped: [{arm,reason}]`,
|
|
45
|
+
// consumed by src/cli/index.mjs, src/cli/serve.mjs and src/mcp/tools.mjs as an iterable) --
|
|
46
|
+
// a same-named sibling with a different shape would break every one of those.
|
|
47
|
+
const embedded = { text: 0, image: 0, skipped: { image: 0 } };
|
|
34
48
|
const refused = [];
|
|
35
49
|
const insEntry = db.prepare(`INSERT INTO entries VALUES (@slug,@title,@tier,@source_kind,@captured_at,@dir,@body,@fm)`);
|
|
36
50
|
const insTag = db.prepare(`INSERT INTO entry_tags VALUES (?,?,?)`);
|
|
@@ -58,26 +72,53 @@ export async function buildIndex({ entriesDir, indexPath, vocab, models, p }) {
|
|
|
58
72
|
tx(e, tags);
|
|
59
73
|
if (emb && db.vec) {
|
|
60
74
|
const tagsStr = tags.map(([f, t]) => `${f}/${t}`).join(' ');
|
|
61
|
-
const
|
|
62
|
-
|
|
75
|
+
const text = e.fm.title + '\n' + searchableBody(e) + '\n' + tagsStr;
|
|
76
|
+
// Keyed on the exact string that gets embedded, so an edit that changes the text changes
|
|
77
|
+
// the key and re-embeds, and an edit that does not (whitespace in an unindexed field)
|
|
78
|
+
// correctly costs nothing.
|
|
79
|
+
const textKey = createHash('sha256').update(text).digest('hex');
|
|
80
|
+
let tv = cache?.get(`text:${emb.id}`, textKey) ?? null;
|
|
81
|
+
if (!tv) {
|
|
82
|
+
tv = Buffer.from((await emb.text(text)).buffer);
|
|
83
|
+
// tv is already computed; a cache write failure (SQLITE_FULL, a broken cache) must
|
|
84
|
+
// cost exactly the write, not the build.
|
|
85
|
+
try { cache?.put(`text:${emb.id}`, textKey, tv); } catch (err) {
|
|
86
|
+
process.stderr.write(`index: ${e.slug}: text vector cache write skipped: ${err.message}\n`);
|
|
87
|
+
}
|
|
88
|
+
embedded.text++;
|
|
89
|
+
}
|
|
90
|
+
insVecText.run(e.slug, tv);
|
|
63
91
|
// An unreachable strip must not abort the whole index: skip this entry's image
|
|
64
92
|
// vector and keep going, the same degradation buildUi applies to its thumbnail.
|
|
65
93
|
const strip = (e.fm.captured?.captures?.[0]?.artifacts ?? []).find((a) => a?.key?.endsWith('/strip.webp'));
|
|
66
94
|
if (strip) {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
95
|
+
// Keyed on the digest entry.md ALREADY records, which is the whole win: on a hit there
|
|
96
|
+
// is no artifactFile call, so no fetch and no cache read -- a restart over an unchanged
|
|
97
|
+
// corpus does zero network I/O for images.
|
|
98
|
+
let iv = cache?.get(`image:${emb.id}`, strip.sha256) ?? null;
|
|
99
|
+
if (!iv) {
|
|
100
|
+
try {
|
|
101
|
+
iv = Buffer.from((await emb.image(await artifactFile(p, strip))).buffer);
|
|
102
|
+
cache?.put(`image:${emb.id}`, strip.sha256, iv);
|
|
103
|
+
embedded.image++;
|
|
104
|
+
} catch (err) {
|
|
105
|
+
iv = null;
|
|
106
|
+
embedded.skipped.image++;
|
|
107
|
+
process.stderr.write(`index: ${e.slug}: image vector skipped: ${err.message}\n`);
|
|
108
|
+
}
|
|
72
109
|
}
|
|
110
|
+
if (iv) insVecImage.run(e.slug, iv);
|
|
73
111
|
}
|
|
74
112
|
}
|
|
75
113
|
n++;
|
|
76
114
|
}
|
|
77
|
-
|
|
115
|
+
// A close failure here must not mask a completed build or discard its index -- same
|
|
116
|
+
// reasoning as the catch block below, extended to the success path.
|
|
117
|
+
try { db.close(); } catch {}
|
|
118
|
+
try { cache?.close(); } catch {}
|
|
78
119
|
fs.renameSync(tmpPath, indexPath);
|
|
79
120
|
return {
|
|
80
|
-
entries: n, refused,
|
|
121
|
+
entries: n, refused, embedded,
|
|
81
122
|
arms: emb && db.vec ? ['fts', 'vec'] : ['fts'],
|
|
82
123
|
skipped: emb
|
|
83
124
|
? (db.vec ? [] : [{ arm: 'vec', reason: 'sqlite-vec failed to load' }])
|
|
@@ -88,6 +129,7 @@ export async function buildIndex({ entriesDir, indexPath, vocab, models, p }) {
|
|
|
88
129
|
// handle here leaks for the life of the process. A close failure must not mask
|
|
89
130
|
// the original error, so it's swallowed rather than rethrown.
|
|
90
131
|
try { db?.close(); } catch {}
|
|
132
|
+
try { cache?.close(); } catch {}
|
|
91
133
|
fs.rmSync(tmpPath, { force: true });
|
|
92
134
|
throw err;
|
|
93
135
|
}
|
package/src/index/embed.mjs
CHANGED
|
@@ -29,7 +29,12 @@ export async function getEmbedder({ models, mode = process.env.LIBRARY_EMBED } =
|
|
|
29
29
|
if (mode === 'off') return null;
|
|
30
30
|
if (mode === 'stub') {
|
|
31
31
|
return {
|
|
32
|
-
|
|
32
|
+
// Namespaced, so no value of LIBRARY_EMBED_STUB_ID can equal the real arm's id below.
|
|
33
|
+
// Unprefixed, setting it to the two model constants verbatim produced a cache key
|
|
34
|
+
// byte-identical to a genuine real-arm build, and a later real build read the stub's
|
|
35
|
+
// hash-noise vectors as a warm hit — the very bug this id exists to prevent, re-entered
|
|
36
|
+
// through its own test seam. Reproduced before this line was added.
|
|
37
|
+
arm: 'vec', dims: DIMS, id: `stub:${process.env.LIBRARY_EMBED_STUB_ID ?? 'default'}`,
|
|
33
38
|
text: async (s) => stubVec('t:' + s, DIMS.text),
|
|
34
39
|
// ponytail: seeded on byte length + basename, not pixel content. Every
|
|
35
40
|
// indexed strip shares the basename strip.webp, so two strips of the same
|
|
@@ -48,7 +53,7 @@ export async function getEmbedder({ models, mode = process.env.LIBRARY_EMBED } =
|
|
|
48
53
|
const proc = await tf.AutoProcessor.from_pretrained(IMAGE_MODEL);
|
|
49
54
|
const vision = await tf.SiglipVisionModel.from_pretrained(IMAGE_MODEL);
|
|
50
55
|
return {
|
|
51
|
-
arm: 'vec', dims: DIMS,
|
|
56
|
+
arm: 'vec', dims: DIMS, id: `${TEXT_MODEL}|${IMAGE_MODEL}`,
|
|
52
57
|
text: async (s) => Float32Array.from((await textPipe(s, { pooling: 'mean', normalize: true })).data),
|
|
53
58
|
image: async (p) => {
|
|
54
59
|
const img = await tf.RawImage.read(p);
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { openIndex } from './query.mjs';
|
|
4
|
+
|
|
5
|
+
const SCHEMA = `
|
|
6
|
+
CREATE TABLE IF NOT EXISTS vectors (
|
|
7
|
+
kind TEXT NOT NULL,
|
|
8
|
+
key TEXT NOT NULL,
|
|
9
|
+
vec BLOB NOT NULL,
|
|
10
|
+
PRIMARY KEY (kind, key)
|
|
11
|
+
);
|
|
12
|
+
`;
|
|
13
|
+
|
|
14
|
+
// A separate file, deliberately. buildIndex writes index.sqlite.tmp and renames it over the real
|
|
15
|
+
// index on every run, so a cache table inside it would be destroyed by the very operation it is
|
|
16
|
+
// meant to make cheap. This one lives beside it and outlives every rebuild.
|
|
17
|
+
export function openVecCache(p) {
|
|
18
|
+
// .library/, beside index.sqlite and media/ — the one place on disk where derived bytes live,
|
|
19
|
+
// and the directory .gitignore covers. Loose in p.data it would be an untracked file in the
|
|
20
|
+
// repo root of every local checkout.
|
|
21
|
+
const file = path.join(p.data, '.library', 'vectors.sqlite');
|
|
22
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
23
|
+
let db;
|
|
24
|
+
try {
|
|
25
|
+
db = openIndex(file, { readonly: false });
|
|
26
|
+
// Unconditional, and deliberately so. A review flagged this as taking a write lock on every
|
|
27
|
+
// open — two processes legitimately share this volume, so that would have mattered. Measured,
|
|
28
|
+
// it does not: with another connection holding BEGIN IMMEDIATE, `CREATE TABLE IF NOT EXISTS`
|
|
29
|
+
// against an EXISTING table succeeds and takes no write lock, and only the first open, when
|
|
30
|
+
// the table really is absent, raises SQLITE_BUSY. Guarding this with a `SELECT name FROM
|
|
31
|
+
// sqlite_master` first would add a query to every open to avoid a lock that is never taken
|
|
32
|
+
// after the first one. Left simple on purpose.
|
|
33
|
+
db.exec(SCHEMA);
|
|
34
|
+
const sel = db.prepare('SELECT vec FROM vectors WHERE kind = ? AND key = ?');
|
|
35
|
+
const ins = db.prepare('INSERT OR REPLACE INTO vectors (kind, key, vec) VALUES (?, ?, ?)');
|
|
36
|
+
return {
|
|
37
|
+
// null means MISS. A row holding a zero-length blob is a hit whose value is empty, and the
|
|
38
|
+
// two must stay distinguishable or an empty vector re-embeds on every single start.
|
|
39
|
+
get(kind, key) {
|
|
40
|
+
const row = sel.get(kind, key);
|
|
41
|
+
return row ? Buffer.from(row.vec) : null;
|
|
42
|
+
},
|
|
43
|
+
put(kind, key, buf) { ins.run(kind, key, buf); },
|
|
44
|
+
close() { db.close(); },
|
|
45
|
+
};
|
|
46
|
+
} catch (err) {
|
|
47
|
+
// openVecCache can be called on every http server startup or in CLI runs on a shared
|
|
48
|
+
// volume; an unclosed handle here leaks for the life of the process. A close failure must
|
|
49
|
+
// not mask the original error, so it's swallowed rather than rethrown.
|
|
50
|
+
try { db?.close(); } catch {}
|
|
51
|
+
throw err;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { parseEntry, writeEntry, REQUIRED_BLANKS } from '../entry.mjs';
|
|
2
|
+
import { resolveTerm } from '../vocab.mjs';
|
|
3
|
+
|
|
4
|
+
// Exactly the fields a session is the author of. tier and ported_from are here
|
|
5
|
+
// because a ported entry cannot be completed without them and the licence gate in
|
|
6
|
+
// validateEntry still judges the result. Everything absent from this list is
|
|
7
|
+
// provenance the server measured, and is refused by name below.
|
|
8
|
+
const ALLOWED = new Set([...REQUIRED_BLANKS, 'tier', 'ported_from']);
|
|
9
|
+
|
|
10
|
+
// Which vocabulary facet governs which field. A field absent here takes free text.
|
|
11
|
+
const FACET_OF = { roles: 'role', feel: 'feel', 'motion.devices': 'motion', tier: 'tier' };
|
|
12
|
+
|
|
13
|
+
function get(obj, dotted) {
|
|
14
|
+
return dotted.split('.').reduce((o, k) => (o == null ? undefined : o[k]), obj);
|
|
15
|
+
}
|
|
16
|
+
function set(obj, dotted, value) {
|
|
17
|
+
const keys = dotted.split('.');
|
|
18
|
+
const last = keys.pop();
|
|
19
|
+
let o = obj;
|
|
20
|
+
for (const k of keys) {
|
|
21
|
+
if (o[k] == null || typeof o[k] !== 'object') o[k] = {};
|
|
22
|
+
o = o[k];
|
|
23
|
+
}
|
|
24
|
+
o[last] = value;
|
|
25
|
+
}
|
|
26
|
+
function isBlank(v) {
|
|
27
|
+
return v == null || v === '' || (Array.isArray(v) && v.length === 0);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function setFields({ dir, fields, vocab }) {
|
|
31
|
+
const entry = parseEntry(dir);
|
|
32
|
+
const fm = entry.fm;
|
|
33
|
+
let body = entry.body;
|
|
34
|
+
const refused = [];
|
|
35
|
+
const staged = [];
|
|
36
|
+
|
|
37
|
+
for (const [key, value] of Object.entries(fields ?? {})) {
|
|
38
|
+
if (!ALLOWED.has(key)) {
|
|
39
|
+
refused.push(`${key}: not a judgement field — set_fields writes ${[...ALLOWED].join(', ')} and nothing else`);
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
const facet = FACET_OF[key];
|
|
43
|
+
if (facet) {
|
|
44
|
+
const terms = Array.isArray(value) ? value : [value];
|
|
45
|
+
const canon = [];
|
|
46
|
+
for (const t of terms) {
|
|
47
|
+
const c = resolveTerm(vocab, facet, t);
|
|
48
|
+
if (c === null) refused.push(`${key}: unknown ${facet} term "${t}"`);
|
|
49
|
+
else canon.push(c);
|
|
50
|
+
}
|
|
51
|
+
staged.push([key, Array.isArray(value) ? canon : canon[0]]);
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
staged.push([key, value]);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Nothing is written when anything was refused. A partial write leaves the agent
|
|
58
|
+
// reasoning about a draft that is neither what it sent nor what it had.
|
|
59
|
+
if (refused.length) {
|
|
60
|
+
return { ok: false, slug: fm.slug, blanks: REQUIRED_BLANKS.filter((k) => isBlank(get({ ...fm, body }, k))), written: [], refused };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
for (const [key, value] of staged) {
|
|
64
|
+
if (key === 'body') body = value;
|
|
65
|
+
else set(fm, key, value);
|
|
66
|
+
}
|
|
67
|
+
writeEntry(dir, fm, body);
|
|
68
|
+
|
|
69
|
+
const view = { ...fm, body };
|
|
70
|
+
return {
|
|
71
|
+
ok: true,
|
|
72
|
+
slug: fm.slug,
|
|
73
|
+
blanks: REQUIRED_BLANKS.filter((k) => isBlank(get(view, k))),
|
|
74
|
+
written: staged.map(([k]) => k),
|
|
75
|
+
refused: [],
|
|
76
|
+
};
|
|
77
|
+
}
|
package/src/mcp/http.mjs
CHANGED
|
@@ -23,8 +23,23 @@ async function handle(req, res, p, token, embedder) {
|
|
|
23
23
|
if (db.vec && db.prepare('SELECT count(*) c FROM vec_text').get().c > 0) arms = ['fts', 'vec'];
|
|
24
24
|
db.close();
|
|
25
25
|
} catch {}
|
|
26
|
+
// A host with a full index and no reachable artifacts serves 31 grey boxes and reported
|
|
27
|
+
// ok:true, which is how a missing R2_PUBLIC_BASE_URL stayed invisible through two rounds of
|
|
28
|
+
// "it still is not showing images". Name it here, where anyone can see it without log access.
|
|
29
|
+
// It is not part of `ok`: the MCP API answers perfectly well on text alone, and failing the
|
|
30
|
+
// health check would take a working server down over missing pictures.
|
|
31
|
+
// "configured", never "ok": this reads the environment, it does not fetch an object, so it
|
|
32
|
+
// cannot claim the bucket is reachable. It catches the failure actually hit — the variable
|
|
33
|
+
// being absent — and says only what it checked.
|
|
34
|
+
let artifacts;
|
|
35
|
+
try {
|
|
36
|
+
const { readConfig } = await import('../r2.mjs');
|
|
37
|
+
artifacts = `configured: ${readConfig().base}`;
|
|
38
|
+
} catch (err) {
|
|
39
|
+
artifacts = `unconfigured: ${err.message}`;
|
|
40
|
+
}
|
|
26
41
|
res.writeHead(200, { 'content-type': 'application/json' });
|
|
27
|
-
return res.end(JSON.stringify({ ok: true, entries, arms, version: p.version }));
|
|
42
|
+
return res.end(JSON.stringify({ ok: true, entries, arms, artifacts, version: p.version }));
|
|
28
43
|
}
|
|
29
44
|
if (url.pathname === '/mcp') {
|
|
30
45
|
if (!authorized(req.headers.authorization, token)) { res.writeHead(401); return res.end('unauthorized'); }
|
package/src/mcp/prompts.mjs
CHANGED
|
@@ -40,8 +40,14 @@ session did not make.
|
|
|
40
40
|
\`frames: 36\` before tagging. The server caps at 36 frames; beyond about 18 seconds a
|
|
41
41
|
section can still be missed, so build a denser sheet yourself from the source file
|
|
42
42
|
before writing the motion note.
|
|
43
|
-
4. **Tag.** Fill every field in the returned \`blanks\` list
|
|
44
|
-
|
|
43
|
+
4. **Tag.** Fill every field in the returned \`blanks\` list, per the rubric below. Write
|
|
44
|
+
the motion note last, after the roster. Two ways, same result: call \`set_fields\` with
|
|
45
|
+
those field names — this needs **no filesystem** and is the only way that works against
|
|
46
|
+
a hosted server — or, in a local checkout, edit \`<dir>/entry.md\` directly.
|
|
47
|
+
\`set_fields\` resolves aliases and refuses an unknown term by name without writing
|
|
48
|
+
anything, so a refusal never leaves a half-filled draft. It writes judgement fields
|
|
49
|
+
only; provenance is refused. \`get_vocab\` returns the terms if you did not read them
|
|
50
|
+
above.
|
|
45
51
|
5. **Approve.** Show the draft to the user and **show the draft and wait for a yes** before
|
|
46
52
|
anything else. Apply their edits.
|
|
47
53
|
6. **Save.** Call \`save_entry\`. On \`refused\`, show the field names, fix, and call again;
|
package/src/mcp/tools.mjs
CHANGED
|
@@ -213,6 +213,15 @@ export function makeTools(ctx) {
|
|
|
213
213
|
return { arms: arms ?? ['fts'], ...(skipped ? { skipped } : {}), feel, roles };
|
|
214
214
|
}),
|
|
215
215
|
},
|
|
216
|
+
{
|
|
217
|
+
name: 'get_vocab',
|
|
218
|
+
description: 'The five controlled facets and the alias map, read from vocab.yaml. roles and feel tag an entry, motion names the data-motion devices, source and tier gate the licence. A term is added by PR, never by an ingest, so an unknown term is a refusal rather than a new term.',
|
|
219
|
+
schema: z.object({}),
|
|
220
|
+
handler: async () => ({
|
|
221
|
+
...Object.fromEntries(Object.entries(ctx.vocab.facets).map(([k, v]) => [k, [...v]])),
|
|
222
|
+
aliases: Object.fromEntries(ctx.vocab.aliases),
|
|
223
|
+
}),
|
|
224
|
+
},
|
|
216
225
|
{
|
|
217
226
|
name: 'add',
|
|
218
227
|
description: 'Ingest a video/animated image, or measure a live https:// URL with the system Chrome: sample frames or measure into strip.webp/measure.json and write a draft entry. Returns the blank fields the caller must fill before save_entry. Pass attach to add a second capture of the same site — a video beside a live URL — instead of refusing; the existing entry\'s tags, body, palette and type are left untouched.',
|
|
@@ -238,6 +247,26 @@ export function makeTools(ctx) {
|
|
|
238
247
|
return (await import('../cli/add.mjs')).addEntry(a);
|
|
239
248
|
},
|
|
240
249
|
},
|
|
250
|
+
{
|
|
251
|
+
name: 'add_batch',
|
|
252
|
+
description: "Ingest every media file in the server's inbox in one run: one draft per file, an existing slug skipped so an interrupted run resumes, and a failed file costing only itself. Returns one record per file. Takes no path — the walk is the inbox and nothing else. Each drafted entry still needs set_fields then save_entry.",
|
|
253
|
+
schema: z.object({
|
|
254
|
+
source: z.object({
|
|
255
|
+
kind: z.enum(['own', 'open', 'paid', 'public-site']),
|
|
256
|
+
url: z.string().url().optional(),
|
|
257
|
+
license: z.string().min(1),
|
|
258
|
+
}).optional(),
|
|
259
|
+
}),
|
|
260
|
+
handler: async ({ source }) => {
|
|
261
|
+
const { addInbox } = await import('../cli/add.mjs');
|
|
262
|
+
const results = await addInbox({ source });
|
|
263
|
+
return {
|
|
264
|
+
results,
|
|
265
|
+
drafted: results.filter((r) => r.status === 'drafted').length,
|
|
266
|
+
refused: results.filter((r) => r.status === 'refused').length,
|
|
267
|
+
};
|
|
268
|
+
},
|
|
269
|
+
},
|
|
241
270
|
{
|
|
242
271
|
name: 'refresh',
|
|
243
272
|
description: 'Re-measure a drafted URL entry (source.url set, captured.method measured|both) with the system Chrome: rewrites measure.json, bumps captured.at, and reindexes. Refuses an entry with no URL or one drafted from frames only.',
|
|
@@ -253,6 +282,18 @@ export function makeTools(ctx) {
|
|
|
253
282
|
return saveEntry({ dir: path.join(ctx.paths.entries, slug), vocab: ctx.vocab, indexPath: ctx.paths.index, entriesDir: ctx.paths.entries, models: ctx.paths.models, p: ctx.paths });
|
|
254
283
|
},
|
|
255
284
|
},
|
|
285
|
+
{
|
|
286
|
+
name: 'set_fields',
|
|
287
|
+
description: "Fill a draft's judgement fields: roles, feel, motion.devices, motion.notes, palette, type, body, tier, ported_from — never provenance. Vocabulary terms are resolved through aliases and an unknown one is refused by name with nothing written, so a refusal never leaves a half-filled draft. Returns the fields still blank. This is the step between add and save_entry, and it is the only one that does not need a filesystem.",
|
|
288
|
+
schema: z.object({
|
|
289
|
+
slug: z.string().regex(/^[a-z0-9-]+$/),
|
|
290
|
+
fields: z.record(z.any()),
|
|
291
|
+
}),
|
|
292
|
+
handler: async ({ slug, fields }) => {
|
|
293
|
+
const { setFields } = await import('../ingest/fields.mjs');
|
|
294
|
+
return setFields({ dir: path.join(ctx.paths.entries, slug), fields, vocab: ctx.vocab });
|
|
295
|
+
},
|
|
296
|
+
},
|
|
256
297
|
];
|
|
257
298
|
}
|
|
258
299
|
|
package/src/r2.mjs
CHANGED
|
@@ -59,15 +59,50 @@ export function sha256(body) {
|
|
|
59
59
|
return createHash('sha256').update(body).digest('hex');
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
+
// Extracted from s3Upload, which built this inline. Two constructions of the same client that can
|
|
63
|
+
// drift apart is exactly the kind of duplication that makes one of them quietly wrong later.
|
|
64
|
+
function clientConfig(config) {
|
|
65
|
+
return { region: 'auto', endpoint: `https://${config.account}.r2.cloudflarestorage.com`,
|
|
66
|
+
credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey } };
|
|
67
|
+
}
|
|
68
|
+
|
|
62
69
|
export async function s3Upload(config) {
|
|
63
70
|
const { S3Client, PutObjectCommand } = await import('@aws-sdk/client-s3');
|
|
64
|
-
const client = new S3Client(
|
|
65
|
-
credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey } });
|
|
71
|
+
const client = new S3Client(clientConfig(config));
|
|
66
72
|
return (args) => client.send(new PutObjectCommand(args));
|
|
67
73
|
}
|
|
68
74
|
|
|
69
|
-
//
|
|
70
|
-
|
|
75
|
+
// Mirrors s3Upload: the SDK is imported lazily so a reader never loads it.
|
|
76
|
+
export async function s3Head(config) {
|
|
77
|
+
const { S3Client, HeadObjectCommand } = await import('@aws-sdk/client-s3');
|
|
78
|
+
const client = new S3Client(clientConfig(config));
|
|
79
|
+
return async ({ Bucket, Key }) => {
|
|
80
|
+
const r = await client.send(new HeadObjectCommand({ Bucket, Key }));
|
|
81
|
+
return { Metadata: r.Metadata ?? {} };
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
86
|
+
|
|
87
|
+
// Only the PUT is retried. The range-fetch verification below is not: a verification failure
|
|
88
|
+
// means the object in the bucket is not what we sent, and repeating the read cannot change that
|
|
89
|
+
// -- it would just delay a refusal that is already correct.
|
|
90
|
+
async function withRetry(fn, retries, sleep) {
|
|
91
|
+
for (let attempt = 1; ; attempt++) {
|
|
92
|
+
try { return await fn(); }
|
|
93
|
+
catch (err) {
|
|
94
|
+
if (attempt >= retries) throw err;
|
|
95
|
+
await sleep(250 * 2 ** (attempt - 1));
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Upload -- or, when the digest already matches what's in the bucket, skip the PUT -- then prove
|
|
101
|
+
// the object is publicly readable and byte-identical before any caller records the key. motion.mjs
|
|
102
|
+
// established this order: an entry must never name an object nobody can read, and that holds for
|
|
103
|
+
// the skip path too. The HEAD above is an authenticated S3-API call against the origin; it proves
|
|
104
|
+
// the bytes are right, not that the public edge serves them, so the skip still pays for a 32-byte
|
|
105
|
+
// range-fetch through R2_PUBLIC_BASE_URL -- cheap next to the full upload it avoided.
|
|
71
106
|
//
|
|
72
107
|
// `immutable` is NOT carried over from motion.mjs with it. That key embeds the digest
|
|
73
108
|
// (motion/<slug>/<id>-<sha256[0:16]>.mp4) so it never names two different bodies; an artifact key
|
|
@@ -78,18 +113,38 @@ export async function s3Upload(config) {
|
|
|
78
113
|
// verification below, which range-fetches through R2_PUBLIC_BASE_URL (the edge): under `immutable`
|
|
79
114
|
// it can compare the new body against a cached old object and throw `public bytes differ` AFTER
|
|
80
115
|
// the object was overwritten. The cost is a re-capture taking up to max-age to reach the edge.
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
CacheControl: 'public, max-age=300', Metadata: { sha256: digest } });
|
|
116
|
+
// The one range-fetch check, run after either a real upload or a skip. Two copies of this would
|
|
117
|
+
// drift -- the whole point of the invariant above is that neither path is allowed to skip it.
|
|
118
|
+
async function verifyPublic({ config, key, body, fetcher, digest, uploaded }) {
|
|
85
119
|
const url = objectUrl(config, key);
|
|
86
120
|
const response = await fetcher(url, { headers: { Range: 'bytes=0-31' }, signal: AbortSignal.timeout(20000) });
|
|
87
121
|
const range = response.headers.get('content-range');
|
|
88
122
|
if (response.status !== 206 || range !== `bytes 0-31/${body.length}`) {
|
|
89
123
|
await response.body?.cancel();
|
|
90
|
-
|
|
124
|
+
const verb = uploaded ? 'uploaded' : 'matched';
|
|
125
|
+
throw new Error(`${verb} ${key}, but public verification failed (HTTP ${response.status}); nothing was recorded`);
|
|
126
|
+
}
|
|
127
|
+
const headBytes = Buffer.from(await response.arrayBuffer());
|
|
128
|
+
if (!headBytes.equals(body.subarray(0, 32))) throw new Error(`public bytes differ for ${key}`);
|
|
129
|
+
return { key, bytes: body.length, sha256: digest, uploaded };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export async function putAndVerify({ config, key, body, contentType, upload, head, fetcher = fetch,
|
|
133
|
+
retries = 3, sleep = defaultSleep }) {
|
|
134
|
+
const digest = sha256(body);
|
|
135
|
+
// Skip a PUT whose bytes are already in the bucket. The digest is compared against the object's
|
|
136
|
+
// own recorded metadata, not against its length: two different objects of the same size would
|
|
137
|
+
// otherwise read as identical, which is the exact mistake the artifact cache made and that
|
|
138
|
+
// 1.0.1 had to fix. An object with NO recorded digest is re-uploaded rather than assumed
|
|
139
|
+
// identical -- anything written before this field existed has none, and guessing would skip a
|
|
140
|
+
// needed write forever.
|
|
141
|
+
if (head) {
|
|
142
|
+
try {
|
|
143
|
+
const existing = await head({ Bucket: config.bucket, Key: key });
|
|
144
|
+
if (existing?.Metadata?.sha256 === digest) return verifyPublic({ config, key, body, fetcher, digest, uploaded: false });
|
|
145
|
+
} catch { /* NotFound, or a HEAD this account cannot make: fall through and upload */ }
|
|
91
146
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
return { key,
|
|
147
|
+
await withRetry(() => upload({ Bucket: config.bucket, Key: key, Body: body, ContentType: contentType,
|
|
148
|
+
CacheControl: 'public, max-age=300', Metadata: { sha256: digest } }), retries, sleep);
|
|
149
|
+
return verifyPublic({ config, key, body, fetcher, digest, uploaded: true });
|
|
95
150
|
}
|
package/src/ui/app.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
// Vanilla, no build step: filters the manifest already rendered into `.card`
|
|
2
2
|
// data attributes. Within one facet, checked boxes OR together; across facets, AND.
|
|
3
|
+
// Filtering sets eligibility only (dataset.filtered); repage(), defined globally by the inline
|
|
4
|
+
// script in index.html loaded before this file, is what decides what's actually painted. Paging
|
|
5
|
+
// stays a view over whatever the filter left eligible, never a filter of its own.
|
|
3
6
|
(function () {
|
|
4
7
|
var grid = document.getElementById('grid');
|
|
5
8
|
var cards = grid ? Array.prototype.slice.call(grid.querySelectorAll('.card')) : [];
|
|
@@ -24,8 +27,9 @@
|
|
|
24
27
|
var have = (card.dataset[facets[i]] || '').split(' ');
|
|
25
28
|
ok = want.some(function (t) { return have.indexOf(t) !== -1; });
|
|
26
29
|
}
|
|
27
|
-
card.
|
|
30
|
+
card.dataset.filtered = ok ? '' : 'out';
|
|
28
31
|
});
|
|
32
|
+
repage();
|
|
29
33
|
}
|
|
30
34
|
|
|
31
35
|
var inputs = document.querySelectorAll('#filters input[type="checkbox"]');
|
package/src/ui/build.mjs
CHANGED
|
@@ -46,7 +46,7 @@ function filterFieldset(facet, terms) {
|
|
|
46
46
|
}
|
|
47
47
|
|
|
48
48
|
function card(row) {
|
|
49
|
-
const img = row.thumbnail ? `<img src="${row.thumbnail}" alt="" loading="lazy" decoding="async">` : `<div class="ph"></div>`;
|
|
49
|
+
const img = row.thumbnail ? `<img src="${row.thumbnail}" alt="" loading="lazy" decoding="async" width="640" height="480">` : `<div class="ph"></div>`;
|
|
50
50
|
const badge = row.draft ? ' <span class="badge">draft</span>' : '';
|
|
51
51
|
const tags = esc([...row.roles, ...row.feel, ...row.motion].join(', '));
|
|
52
52
|
const data = `data-role="${esc(row.roles.join(' '))}" data-feel="${esc(row.feel.join(' '))}" data-motion="${esc(row.motion.join(' '))}"`;
|
|
@@ -150,12 +150,20 @@ function writeEntryPage(outDir, entry, row, previewMediaDir, measureFile) {
|
|
|
150
150
|
export async function buildUi({ indexPath, entriesDir, outDir, previewMediaDir, p }) {
|
|
151
151
|
void indexPath;
|
|
152
152
|
const vocab = loadVocab();
|
|
153
|
-
|
|
153
|
+
// Wipe the HTML, keep the pixels. Re-encoding 2000 unchanged thumbnails on every deploy is the
|
|
154
|
+
// single most expensive thing this build does, and a thumbnail is a pure function of the strip
|
|
155
|
+
// digest already recorded in entry.md — so it is safe to keep and cheap to invalidate.
|
|
156
|
+
for (const name of fs.existsSync(outDir) ? fs.readdirSync(outDir) : []) {
|
|
157
|
+
if (name === 'thumbnails' || name === 'strips') continue;
|
|
158
|
+
fs.rmSync(path.join(outDir, name), { recursive: true, force: true });
|
|
159
|
+
}
|
|
154
160
|
fs.mkdirSync(path.join(outDir, 'entry'), { recursive: true });
|
|
155
161
|
fs.mkdirSync(path.join(outDir, 'strips'), { recursive: true });
|
|
156
162
|
fs.mkdirSync(path.join(outDir, 'thumbnails'), { recursive: true });
|
|
157
163
|
|
|
158
164
|
const rows = [];
|
|
165
|
+
const unresolved = [];
|
|
166
|
+
let rendered = 0;
|
|
159
167
|
for (const dir of listEntries(entriesDir)) {
|
|
160
168
|
let entry;
|
|
161
169
|
try { entry = parseEntry(dir); } catch (err) { process.stderr.write(`ui: skipping ${path.basename(dir)}: ${err.message}\n`); continue; }
|
|
@@ -167,21 +175,45 @@ export async function buildUi({ indexPath, entriesDir, outDir, previewMediaDir,
|
|
|
167
175
|
const slug = path.basename(dir);
|
|
168
176
|
let stripSrc = null;
|
|
169
177
|
const strip = (entry.fm.captured?.captures?.[0]?.artifacts ?? []).find((a) => a?.key?.endsWith('/strip.webp'));
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
|
|
173
|
-
|
|
178
|
+
// The index is a browsing surface. Its previews must not make each visitor
|
|
179
|
+
// download a multi-frame source sheet; the full strip remains on the detail page.
|
|
180
|
+
const thumb = path.join(outDir, 'thumbnails', `${slug}.webp`);
|
|
181
|
+
const stamp = path.join(outDir, 'thumbnails', `${slug}.sha256`);
|
|
182
|
+
const stripOut = path.join(outDir, 'strips', `${slug}.webp`);
|
|
183
|
+
// The sidecar records WHICH strip produced this thumbnail, AND which encoder made it: a strip
|
|
184
|
+
// whose digest is unchanged still needs a new render once the encode below changes shape, or
|
|
185
|
+
// a deploy that ships a new encoder serves the old pixels forever. THUMB sits next to the
|
|
186
|
+
// sharp() call it describes (below) so the two cannot drift apart silently -- bump it when
|
|
187
|
+
// that call's parameters change.
|
|
188
|
+
const THUMB = '640x480-cover-north-webp-q72-e4';
|
|
189
|
+
// Consulted BEFORE artifactFile, not after: the selective wipe above preserves thumbnails/
|
|
190
|
+
// and strips/, so when the stamp matches and both outputs are already on disk there is
|
|
191
|
+
// nothing to resolve -- no cache read, no hash, no fetch, for this entry's strip.
|
|
192
|
+
let current = null;
|
|
193
|
+
try { current = fs.readFileSync(stamp, 'utf8').trim(); } catch {}
|
|
194
|
+
const sidecarHit = !!strip && current === `${THUMB}:${strip.sha256}` && fs.existsSync(thumb) && fs.existsSync(stripOut);
|
|
195
|
+
// Same treatment for measure.json: a record to resolve, never a file next to entry.md. It has
|
|
196
|
+
// no sidecar of its own and is a different artifact, so it always resolves here regardless of
|
|
197
|
+
// the strip's sidecar outcome.
|
|
174
198
|
let measureSrc = null;
|
|
175
199
|
const measured = (entry.fm.captured?.captures?.[0]?.artifacts ?? []).find((a) => a?.key?.endsWith('/measure.json'));
|
|
176
|
-
if (measured) { try { measureSrc = await artifactFile(p, measured); } catch { measureSrc = null; } }
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
200
|
+
if (measured) { try { measureSrc = await artifactFile(p, measured); } catch (err) { measureSrc = null; unresolved.push(err); } }
|
|
201
|
+
// One unreachable object must not take the gallery down with it: the entry renders without a
|
|
202
|
+
// thumbnail, exactly as a draft does, and the build continues.
|
|
203
|
+
// Degrading silently is what made a misconfigured host indistinguishable from a broken one:
|
|
204
|
+
// a gallery of 31 grey boxes, `ok: true` from healthz, and not one line saying why. Swallow
|
|
205
|
+
// the failure, still — but count it and keep the first reason.
|
|
206
|
+
if (strip && !sidecarHit) {
|
|
207
|
+
try {
|
|
208
|
+
stripSrc = await artifactFile(p, strip);
|
|
209
|
+
fs.copyFileSync(stripSrc, stripOut);
|
|
210
|
+
await sharp(stripSrc).resize(640, 480, { fit: 'cover', position: 'north' }) // THUMB above must name these params
|
|
211
|
+
.webp({ quality: 72, effort: 4 }).toFile(thumb);
|
|
212
|
+
fs.writeFileSync(stamp, `${THUMB}:${strip.sha256}`);
|
|
213
|
+
rendered++;
|
|
214
|
+
} catch (err) { stripSrc = null; unresolved.push(err); }
|
|
184
215
|
}
|
|
216
|
+
const hasStrip = sidecarHit || stripSrc !== null;
|
|
185
217
|
const fm = entry.fm;
|
|
186
218
|
const capturedAt = fm.captured?.at;
|
|
187
219
|
const captured = capturedAt instanceof Date ? capturedAt.toISOString().slice(0, 10) : String(capturedAt ?? '');
|
|
@@ -212,5 +244,11 @@ export async function buildUi({ indexPath, entriesDir, outDir, previewMediaDir,
|
|
|
212
244
|
fs.writeFileSync(path.join(outDir, 'index.html'), render(tpl, { FILTERS: filters, GRID: rows.map(card).join('\n'), NOTICE: notice }));
|
|
213
245
|
fs.copyFileSync(path.join(DIR, 'app.js'), path.join(outDir, 'app.js'));
|
|
214
246
|
|
|
215
|
-
|
|
247
|
+
// One line, not one per artifact: 31 entries failing for the same missing variable is one
|
|
248
|
+
// fact, and printing it 62 times would bury it. The first reason is the one that matters —
|
|
249
|
+
// they are identical when the cause is configuration, which is the case worth naming.
|
|
250
|
+
if (unresolved.length) {
|
|
251
|
+
process.stderr.write(`ui: ${unresolved.length} artifact(s) could not be resolved, so those entries render without a thumbnail; first reason: ${unresolved[0].message}\n`);
|
|
252
|
+
}
|
|
253
|
+
return { entries: rows.length - drafts, drafts, unresolved: unresolved.length, rendered };
|
|
216
254
|
}
|
|
@@ -26,9 +26,32 @@
|
|
|
26
26
|
<label><input type="checkbox" id="drafts-toggle"> Show drafts</label>
|
|
27
27
|
</form>
|
|
28
28
|
{{NOTICE}}
|
|
29
|
-
<div id="grid">
|
|
29
|
+
<div id="grid" data-page-size="60">
|
|
30
30
|
{{GRID}}
|
|
31
31
|
</div>
|
|
32
|
+
<button id="show-more" hidden>Show more</button>
|
|
33
|
+
<script>
|
|
34
|
+
// Paging is a view over whatever the filter left eligible, never a filter itself: `cards` is
|
|
35
|
+
// still every card in the document, so a search always sees the whole corpus. Only how many
|
|
36
|
+
// eligible cards are currently painted changes.
|
|
37
|
+
var cards = document.querySelectorAll(".card");
|
|
38
|
+
var PAGE = parseInt(document.querySelector("[data-page-size]").dataset.pageSize, 10);
|
|
39
|
+
var shown = PAGE;
|
|
40
|
+
function repage() {
|
|
41
|
+
var n = 0;
|
|
42
|
+
cards.forEach(function (c) {
|
|
43
|
+
if (c.dataset.filtered === "out") { c.hidden = true; return; }
|
|
44
|
+
n++;
|
|
45
|
+
c.hidden = n > shown;
|
|
46
|
+
});
|
|
47
|
+
var more = document.getElementById("show-more");
|
|
48
|
+
more.hidden = n <= shown;
|
|
49
|
+
}
|
|
50
|
+
document.getElementById("show-more").addEventListener("click", function () {
|
|
51
|
+
shown += PAGE;
|
|
52
|
+
repage();
|
|
53
|
+
});
|
|
54
|
+
</script>
|
|
32
55
|
<script src="app.js"></script>
|
|
33
56
|
</body>
|
|
34
57
|
</html>
|