@yojahny/wp-design-library 1.0.1 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -172,6 +172,15 @@ blanks. The same works from the plugin: with the MCP server registered, say
172
172
  "add `~/Desktop/site.webp` to the design library as `acme-dark-hero`" and the
173
173
  session calls `add`, reads the strip, fills the draft, and calls `save_entry`.
174
174
 
175
+ Ingest needs no filesystem: `add` → `set_fields` → `save_entry` closes entirely over
176
+ MCP, so it works against a hosted instance as well as a local checkout. `set_fields`
177
+ writes judgement fields only — roles, feel, motion, palette, type, body, tier — and
178
+ refuses provenance keys by name, so `captured.*` and the artifact digests stay the
179
+ server's record of what it measured. `get_vocab` returns the controlled terms, and
180
+ `add_batch` drafts every file in the server's inbox in one run, resuming where an
181
+ interrupted run stopped. Editing `entries/<slug>/entry.md` by hand remains available in
182
+ a local checkout and does the same thing.
183
+
175
184
  What goes where, with the values checked against `vocab.yaml`:
176
185
 
177
186
  | Field | What to write |
@@ -248,6 +257,7 @@ The plugin does this for you, but from the CLI:
248
257
  ```bash
249
258
  node bin/library.mjs index # rebuild the index from entries/
250
259
  node bin/library.mjs check # validate every entry, 1 line per problem
260
+ node bin/library.mjs check --remote # + confirm every recorded artifact key resolves in R2
251
261
  ```
252
262
 
253
263
  Over MCP, seven tools: `search` (keyword + facet filters, aliases resolved),
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@yojahny/wp-design-library",
3
- "version": "1.0.1",
3
+ "version": "1.1.0",
4
4
  "description": "Design reference corpus and MCP server for claude-wp-builder",
5
5
  "repository": {
6
6
  "type": "git",
package/src/artifacts.mjs CHANGED
@@ -17,7 +17,15 @@ export function cachePath(p, key) {
17
17
  // migrate-artifacts.mjs documents at length; the fix had not reached this file.
18
18
  export async function artifactFile(p, artifact, { config, fetcher = fetch } = {}) {
19
19
  const file = cachePath(p, artifact.key);
20
- if (fs.existsSync(file) && fs.statSync(file).size === artifact.bytes) return file;
20
+ // bytes is the cheap first gate -- it short-circuits the common mismatch without hashing. A
21
+ // length match is not proof: bytes can rewrite in place at the same length, so a HIT is now
22
+ // verified against the digest too. A hit that fails is a corrupt cache entry, not a dead
23
+ // reader -- self-heal by deleting it and falling through to the fetch below, the same path a
24
+ // genuine miss takes.
25
+ if (fs.existsSync(file) && fs.statSync(file).size === artifact.bytes) {
26
+ if (sha256(fs.readFileSync(file)) === artifact.sha256) return file;
27
+ fs.rmSync(file);
28
+ }
21
29
  config ??= readConfig();
22
30
  const url = objectUrl(config, artifact.key);
23
31
  const response = await fetcher(url, { signal: AbortSignal.timeout(30000) });
package/src/cli/add.mjs CHANGED
@@ -8,7 +8,7 @@ import { attachCapture } from '../ingest/attach.mjs';
8
8
  import { measure } from '../ingest/measure.mjs';
9
9
  import { loadVocab, resolveTerm } from '../vocab.mjs';
10
10
  import { parseEntry } from '../entry.mjs';
11
- import { artifactKey, putAndVerify, r2Config, s3Upload } from '../r2.mjs';
11
+ import { artifactKey, putAndVerify, r2Config, s3Head, s3Upload } from '../r2.mjs';
12
12
 
13
13
  function opt(args, name) { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; }
14
14
 
@@ -43,7 +43,7 @@ async function stripFromFrames(framePaths, out) {
43
43
  // URL/file:// input: measure the live page with system Chrome instead of sampling
44
44
  // frames from a recording. Scratch-dir-first, same as the frame path: nothing under
45
45
  // entries/ is created OR destroyed until every byte is captured AND uploaded.
46
- async function addFromUrl({ p, input, slug, title, source, dir, record, force, attach, config, upload, fetcher }) {
46
+ async function addFromUrl({ p, input, slug, title, source, dir, record, force, attach, config, upload, head, fetcher }) {
47
47
  const fs = await import('node:fs');
48
48
  const os = await import('node:os');
49
49
  const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'wpdl-add-'));
@@ -88,8 +88,8 @@ async function addFromUrl({ p, input, slug, title, source, dir, record, force, a
88
88
  // network round trip, and it throws for reasons this process does not control (absent
89
89
  // credentials, a reset connection, a digest mismatch). Uploading after the wipe made every one
90
90
  // of those a data-loss bug: `--force` deleted the entry and put nothing in its place.
91
- // config/upload/fetcher are resolved lazily, here, and not as addEntry's defaults, so a refusal
92
- // earlier in the flow never needs R2 credentials.
91
+ // config/upload/head/fetcher are resolved lazily, here, and not as addEntry's defaults, so a
92
+ // refusal earlier in the flow never needs R2 credentials.
93
93
  const names = ['strip.webp'];
94
94
  if (result.ok) names.push('measure.json', 'screen-1440.png', 'screen-390.png');
95
95
  const types = { '.webp': 'image/webp', '.png': 'image/png', '.json': 'application/json' };
@@ -97,10 +97,11 @@ async function addFromUrl({ p, input, slug, title, source, dir, record, force, a
97
97
  try {
98
98
  config ??= r2Config();
99
99
  if (!upload) upload = await s3Upload(config);
100
+ if (!head) head = await s3Head(config);
100
101
  for (const name of names) {
101
102
  const body = fs.readFileSync(path.join(scratch, name));
102
103
  const key = artifactKey(slug, index, name);
103
- artifacts.push(await putAndVerify({ config, key, body, contentType: types[path.extname(name)], upload, fetcher }));
104
+ artifacts.push(await putAndVerify({ config, key, body, contentType: types[path.extname(name)], upload, head, fetcher }));
104
105
  }
105
106
  } catch (err) {
106
107
  fs.rmSync(scratch, { recursive: true, force: true });
@@ -131,7 +132,7 @@ async function addFromUrl({ p, input, slug, title, source, dir, record, force, a
131
132
  return { dir: d.dir, strip, blanks: d.blanks, measured: result.ok };
132
133
  }
133
134
 
134
- export async function addEntry({ input, slug, title, source, force = false, frames, record = false, attach = false, config, upload, fetcher = fetch }) {
135
+ export async function addEntry({ input, slug, title, source, force = false, frames, record = false, attach = false, config, upload, head, fetcher = fetch }) {
135
136
  const p = paths();
136
137
  if (!input || !slug || !title || !source?.kind || !source?.license) throw new Error('add: input, slug, title, --source and --license are required');
137
138
  // Refused here (draft time), not first at save: resolveTerm is the single vocab gate
@@ -147,7 +148,7 @@ export async function addEntry({ input, slug, title, source, force = false, fram
147
148
  return { refused: `entry ${slug} exists; pass attach to add a capture, or force to overwrite` };
148
149
  }
149
150
  if (URL_RE.test(input)) {
150
- return addFromUrl({ p, input, slug, title, source, dir, record, force, attach, config, upload, fetcher });
151
+ return addFromUrl({ p, input, slug, title, source, dir, record, force, attach, config, upload, head, fetcher });
151
152
  }
152
153
  // Sample into a scratch dir first: a refusal must leave nothing under entries/.
153
154
  const scratch = mkdtempSync(path.join(os.tmpdir(), 'wpdl-add-'));
@@ -170,9 +171,10 @@ export async function addEntry({ input, slug, title, source, force = false, fram
170
171
  try {
171
172
  config ??= r2Config();
172
173
  if (!upload) upload = await s3Upload(config);
174
+ if (!head) head = await s3Head(config);
173
175
  stripArtifact = await putAndVerify({
174
176
  config, key: artifactKey(slug, index, 'strip.webp'), body: readFileSync(scratchStrip),
175
- contentType: 'image/webp', upload, fetcher,
177
+ contentType: 'image/webp', upload, head, fetcher,
176
178
  });
177
179
  } catch (err) {
178
180
  rmSync(scratch, { recursive: true, force: true });
@@ -214,30 +216,40 @@ export async function addEntry({ input, slug, title, source, force = false, fram
214
216
  return { dir: d.dir, strip, blanks: d.blanks, frames: media.frames };
215
217
  }
216
218
 
217
- export async function addInbox({ source, config, upload, fetcher }) {
219
+ export async function addInbox({ source, config, upload, head, fetcher }) {
218
220
  const p = paths();
219
221
  const fs = await import('node:fs');
220
222
  const inbox = path.join(p.data, 'inbox');
221
223
  const out = [];
222
224
  if (!fs.existsSync(inbox)) return out;
225
+ // One file per run, not one shared log: a settled 2,000-file inbox re-run appends 2,000 more
226
+ // `skipped` lines to a file that never shrinks, every time. The ISO stamp also sorts by name,
227
+ // so "what did the run that died last night do" is answerable by reading the newest file.
228
+ const runLog = path.join(p.data, '.library', 'runs', `inbox-${new Date().toISOString().replace(/[:.]/g, '-')}.jsonl`);
229
+ fs.mkdirSync(path.dirname(runLog), { recursive: true });
230
+ // Appended as each file finishes, not written at the end: a run killed at entry 1,400 never
231
+ // reaches its own return statement, and that is the case this exists for. One complete line per
232
+ // append, so a process killed mid-write loses at most its last record rather than corrupting the
233
+ // file. .library/ is already gitignored.
234
+ const record = (r) => { fs.appendFileSync(runLog, JSON.stringify({ ...r, at: new Date().toISOString() }) + '\n'); return r; };
223
235
  const seenSlugs = new Map(); // ponytail: slug -> first file this run, names the collision
224
236
  for (const file of fs.readdirSync(inbox).filter((f) => MEDIA.test(f)).sort()) {
225
237
  const slug = slugify(file);
226
- if (!slug) { out.push({ file, slug, status: 'refused', reason: 'cannot derive a slug from the file name; pass --slug' }); continue; }
227
- if (seenSlugs.has(slug)) { out.push({ file, slug, status: 'refused', reason: `slug ${slug} collides with ${seenSlugs.get(slug)} in this run` }); continue; }
238
+ if (!slug) { out.push(record({ file, slug, status: 'refused', reason: 'cannot derive a slug from the file name; pass --slug' })); continue; }
239
+ if (seenSlugs.has(slug)) { out.push(record({ file, slug, status: 'refused', reason: `slug ${slug} collides with ${seenSlugs.get(slug)} in this run` })); continue; }
228
240
  seenSlugs.set(slug, file);
229
241
  const full = path.join(inbox, file);
230
- if (fs.existsSync(path.join(p.entries, slug, 'entry.md'))) { out.push({ file, slug, status: 'skipped', reason: 'entry exists' }); continue; }
242
+ if (fs.existsSync(path.join(p.entries, slug, 'entry.md'))) { out.push(record({ file, slug, status: 'skipped', reason: 'entry exists' })); continue; }
231
243
  let meta = {};
232
244
  const side = full + '.json';
233
- if (fs.existsSync(side)) { try { meta = JSON.parse(fs.readFileSync(side, 'utf8')); } catch (e) { out.push({ file, slug, status: 'refused', reason: 'sidecar is not JSON' }); continue; } }
234
- if (meta === null || typeof meta !== 'object' || Array.isArray(meta)) { out.push({ file, slug, status: 'refused', reason: 'sidecar must be a JSON object' }); continue; }
245
+ if (fs.existsSync(side)) { try { meta = JSON.parse(fs.readFileSync(side, 'utf8')); } catch (e) { out.push(record({ file, slug, status: 'refused', reason: 'sidecar is not JSON' })); continue; } }
246
+ if (meta === null || typeof meta !== 'object' || Array.isArray(meta)) { out.push(record({ file, slug, status: 'refused', reason: 'sidecar must be a JSON object' })); continue; }
235
247
  const kind = meta.kind ?? source?.kind, license = meta.license ?? source?.license;
236
- if (!kind || !license) { out.push({ file, slug, status: 'refused', reason: 'no source kind/license: add a <file>.json sidecar or pass --source and --license' }); continue; }
248
+ if (!kind || !license) { out.push(record({ file, slug, status: 'refused', reason: 'no source kind/license: add a <file>.json sidecar or pass --source and --license' })); continue; }
237
249
  try {
238
- const r = await addEntry({ input: full, slug, title: meta.title ?? slug.replace(/-/g, ' '), source: { kind, url: meta.url ?? source?.url, license }, config, upload, fetcher });
239
- out.push(r.refused ? { file, slug, status: 'refused', reason: r.refused } : { file, slug, status: 'drafted', frames: r.frames });
240
- } catch (e) { out.push({ file, slug, status: 'refused', reason: e.message }); }
250
+ const r = await addEntry({ input: full, slug, title: meta.title ?? slug.replace(/-/g, ' '), source: { kind, url: meta.url ?? source?.url, license }, config, upload, head, fetcher });
251
+ out.push(record(r.refused ? { file, slug, status: 'refused', reason: r.refused } : { file, slug, status: 'drafted', frames: r.frames }));
252
+ } catch (e) { out.push(record({ file, slug, status: 'refused', reason: e.message })); }
241
253
  }
242
254
  return out;
243
255
  }
package/src/cli/check.mjs CHANGED
@@ -4,6 +4,17 @@ import { loadVocab } from '../vocab.mjs';
4
4
  import { listEntries, parseEntry, validateEntry } from '../entry.mjs';
5
5
  import { fetchModels, modelDirs } from '../index/embed.mjs';
6
6
 
7
+ // A fixed-width worker pool, not one request at a time: ~2000 entries at up to four artifacts
8
+ // each is ~8000 HEADs, and sequential that is minutes at a realistic edge RTT and tens of hours
9
+ // against a bucket that black-holes connections (each request burning the full timeout below). 8
10
+ // is a guess, not tuned -- enough to matter, low enough that a real outage's failures still read
11
+ // as a report instead of a wall of interleaved errors.
12
+ async function mapPool(items, limit, fn) {
13
+ let i = 0;
14
+ const worker = async () => { while (i < items.length) { const idx = i++; await fn(items[idx], idx); } };
15
+ await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
16
+ }
17
+
7
18
  export async function run(args = []) {
8
19
  const p = paths();
9
20
  if (args.includes('--fetch-models')) {
@@ -15,13 +26,47 @@ export async function run(args = []) {
15
26
  }
16
27
  const vocab = loadVocab(p.vocab);
17
28
  let bad = 0, n = 0;
29
+ const artifacts = [];
18
30
  for (const dir of listEntries(p.entries)) {
19
31
  n++;
20
32
  let e;
21
33
  try { e = parseEntry(dir); } catch (err) { bad++; process.stderr.write(`${path.basename(dir)}: ${err.message}\n`); continue; }
22
34
  const r = validateEntry(e, vocab);
23
35
  if (!r.ok) { bad++; for (const m of r.errors) process.stderr.write(`${e.slug}: ${m}\n`); }
36
+ for (const c of e.fm.captured?.captures ?? []) for (const a of c.artifacts ?? []) artifacts.push(a);
24
37
  }
25
38
  process.stdout.write(`${n - bad}/${n} entries ok\n`);
39
+
40
+ // --remote answers the question `check` otherwise never asks: does every key this corpus
41
+ // CLAIMS is in R2 actually resolve. Metadata validation above never touches the network --
42
+ // that stays the default, so CI with no credentials keeps working -- this is opt-in.
43
+ if (args.includes('--remote')) {
44
+ const { readConfig, objectUrl } = await import('../r2.mjs');
45
+ // readConfig throws when R2_PUBLIC_BASE_URL is absent, and unguarded that reaches the operator
46
+ // as a raw Node stack trace -- the same "work it out from the wreckage" failure 1.0.1 fixed for
47
+ // the gallery. It needs only that one variable and no credentials, so the refusal can say
48
+ // exactly what to set.
49
+ let config;
50
+ try { config = readConfig(); }
51
+ catch (err) { process.stderr.write(`check --remote: ${err.message}\n`); return 1; }
52
+ // Collected and sorted before printing, not written as each HEAD lands -- 8 in flight would
53
+ // otherwise interleave and print in whatever order they happen to settle, so the same broken
54
+ // corpus would read differently run to run.
55
+ const problems = [];
56
+ await mapPool(artifacts, 8, async (a) => {
57
+ try {
58
+ const r = await fetch(objectUrl(config, a.key), { method: 'HEAD', signal: AbortSignal.timeout(20000) });
59
+ // Existence AND size. A 200 with the wrong length means the object is not what the entry
60
+ // records -- exactly as broken as it being absent, and cheaper to catch here than at read
61
+ // time on a visitor's page.
62
+ if (!r.ok) { problems.push(`${a.key}: HTTP ${r.status}`); return; }
63
+ const len = Number(r.headers.get('content-length'));
64
+ if (len !== a.bytes) problems.push(`${a.key}: expected ${a.bytes} bytes, bucket says ${len}`);
65
+ } catch (err) { problems.push(`${a.key}: ${err.message}`); }
66
+ });
67
+ for (const p of problems.sort()) process.stderr.write(`${p}\n`);
68
+ process.stdout.write(`${artifacts.length - problems.length}/${artifacts.length} artifacts reachable\n`);
69
+ if (problems.length) return 1;
70
+ }
26
71
  return bad ? 1 : 0;
27
72
  }
package/src/cli/index.mjs CHANGED
@@ -6,7 +6,7 @@ export async function run() {
6
6
  const p = paths();
7
7
  const r = await buildIndex({ entriesDir: p.entries, indexPath: p.index, vocab: loadVocab(p.vocab), models: p.models, p });
8
8
  for (const m of r.refused) process.stderr.write(`refused ${m}\n`);
9
- process.stdout.write(`indexed ${r.entries} entries, refused: ${r.refused.length}, arms: ${r.arms.join(',')}\n`);
9
+ process.stdout.write(`indexed ${r.entries} entries, refused: ${r.refused.length}, arms: ${r.arms.join(',')}, embedded ${r.embedded.text} text and ${r.embedded.image} image vectors\n`);
10
10
  for (const s of r.skipped) process.stderr.write(`embeddings: skipped (${s.reason})\n`);
11
11
  return 0;
12
12
  }
@@ -8,7 +8,7 @@ import { loadVocab } from '../vocab.mjs';
8
8
  import { buildIndex } from '../index/build.mjs';
9
9
  import { measure } from '../ingest/measure.mjs';
10
10
  import { refuseUnsafeUrl } from '../ingest/url-guard.mjs';
11
- import { artifactKey, putAndVerify, r2Config, s3Upload } from '../r2.mjs';
11
+ import { artifactKey, putAndVerify, r2Config, s3Head, s3Upload } from '../r2.mjs';
12
12
 
13
13
  const REFRESHED = ['measure.json', 'screen-1440.png', 'screen-390.png'];
14
14
  const TYPES = { '.png': 'image/png', '.json': 'application/json' };
@@ -19,7 +19,7 @@ const TYPES = { '.png': 'image/png', '.json': 'application/json' };
19
19
  // the measurement and captured.at change. `transport` is undefined for the CLI, 'http'
20
20
  // for the hosted MCP server; the SSRF guard only consults DNS in the latter case.
21
21
  // `measurer` exists so this path can be driven without a browser; nothing but a test passes it.
22
- export async function refreshEntry({ slug, transport, config, upload, fetcher = fetch, measurer = measure }) {
22
+ export async function refreshEntry({ slug, transport, config, upload, head, fetcher = fetch, measurer = measure }) {
23
23
  const p = paths();
24
24
  const dir = path.join(p.entries, slug);
25
25
  let e;
@@ -61,10 +61,11 @@ export async function refreshEntry({ slug, transport, config, upload, fetcher =
61
61
  try {
62
62
  config ??= r2Config();
63
63
  if (!upload) upload = await s3Upload(config);
64
+ if (!head) head = await s3Head(config);
64
65
  uploaded = [];
65
66
  for (const f of REFRESHED) {
66
67
  uploaded.push(await putAndVerify({ config, key: artifactKey(slug, 1, f), body: fs.readFileSync(path.join(scratch, f)),
67
- contentType: TYPES[path.extname(f)], upload, fetcher }));
68
+ contentType: TYPES[path.extname(f)], upload, head, fetcher }));
68
69
  }
69
70
  } catch (err) {
70
71
  fs.rmSync(scratch, { recursive: true, force: true });
package/src/cli/serve.mjs CHANGED
@@ -66,7 +66,7 @@ async function buildTheIndex(p) {
66
66
  const { loadVocab } = await import('../vocab.mjs');
67
67
  const { buildIndex } = await import('../index/build.mjs');
68
68
  const r = await buildIndex({ entriesDir: p.entries, indexPath: p.index, vocab: loadVocab(p.vocab), models: p.models, p });
69
- process.stderr.write(`index: ${r.entries} entries, refused ${r.refused.length}\n`);
69
+ process.stderr.write(`index: ${r.entries} entries, refused ${r.refused.length}, arms: ${r.arms.join(',')}, embedded ${r.embedded.text} text and ${r.embedded.image} image vectors\n`);
70
70
  for (const s of r.skipped) process.stderr.write(`embeddings: skipped (${s.reason})\n`);
71
71
  }
72
72
 
@@ -93,7 +93,8 @@ export async function run(args) {
93
93
  await buildTheIndex(p);
94
94
  try {
95
95
  const { buildUi } = await import('../ui/build.mjs');
96
- await buildUi({ indexPath: p.index, entriesDir: p.entries, outDir: path.join(p.data, 'ui'), p });
96
+ const r = await buildUi({ indexPath: p.index, entriesDir: p.entries, outDir: path.join(p.data, 'ui'), p });
97
+ process.stderr.write(`ui: ${r.entries} entries, ${r.drafts} drafts, rendered ${r.rendered} thumbnails\n`);
97
98
  } catch (err) {
98
99
  // the gallery is a convenience for a human; the MCP API must not depend on it
99
100
  process.stderr.write(`ui: build failed, serving without it: ${err.stack || err}\n`);
package/src/cli/ui.mjs CHANGED
@@ -9,7 +9,7 @@ export async function run(args = []) {
9
9
  const outDir = path.join(p.data, 'ui');
10
10
  const previewMediaDir = args.includes('--local-media') ? path.join(p.data, '.library', 'media') : undefined;
11
11
  const r = await buildUi({ indexPath: p.index, entriesDir: p.entries, outDir, previewMediaDir, p });
12
- process.stdout.write(`ui: ${r.entries} entries, ${r.drafts} drafts, built into ${outDir}\n`);
12
+ process.stdout.write(`ui: ${r.entries} entries, ${r.drafts} drafts, built into ${outDir}, rendered ${r.rendered} thumbnails\n`);
13
13
  if (args.includes('--serve')) {
14
14
  const port = Number(opt(args, '--port') ?? 4180);
15
15
  const { serveStatic } = await import('../ui/serve.mjs');
package/src/entry.mjs CHANGED
@@ -19,6 +19,15 @@ export function parseEntry(dir) {
19
19
  return { slug: fm.slug, dir, fm, body: m[2].trim() };
20
20
  }
21
21
 
22
+ // The symmetric partner to parseEntry. Four other call sites serialize an entry
23
+ // inline with their own lineWidth; they are deliberately left alone, because
24
+ // changing a writer reflows YAML in files it was not asked to touch. New code
25
+ // uses this one.
26
+ export function writeEntry(dir, fm, body) {
27
+ const text = `---\n${yaml.dump(fm, { lineWidth: 120 })}---\n${body ? `\n${body}\n` : ''}`;
28
+ fs.writeFileSync(path.join(dir, 'entry.md'), text);
29
+ }
30
+
22
31
  function get(obj, dotted) {
23
32
  return dotted.split('.').reduce((o, k) => (o == null ? undefined : o[k]), obj);
24
33
  }
@@ -1,10 +1,12 @@
1
1
  import fs from 'node:fs';
2
2
  import path from 'node:path';
3
3
  import { fileURLToPath } from 'node:url';
4
+ import { createHash } from 'node:crypto';
4
5
  import { listEntries, parseEntry, validateEntry } from '../entry.mjs';
5
6
  import { openIndex } from './query.mjs';
6
7
  import { getEmbedder, absentReason } from './embed.mjs';
7
8
  import { artifactFile } from '../artifacts.mjs';
9
+ import { openVecCache } from './veccache.mjs';
8
10
 
9
11
  const DIR = path.dirname(fileURLToPath(import.meta.url));
10
12
  const SCHEMA = fs.readFileSync(path.join(DIR, 'schema.sql'), 'utf8');
@@ -26,11 +28,23 @@ export async function buildIndex({ entriesDir, indexPath, vocab, models, p }) {
26
28
  // a killed build can leave a half-written .tmp behind; clear it so the open below always starts fresh
27
29
  fs.rmSync(tmpPath, { force: true });
28
30
  let db;
31
+ let cache = null;
29
32
  try {
30
33
  db = openIndex(tmpPath, { readonly: false });
31
34
  db.exec(SCHEMA);
32
35
  if (db.vec) db.exec(SCHEMA_VEC);
33
36
  const emb = await getEmbedder({ models });
37
+ // The cache is only worth opening when there is an embedder and a vec arm to feed.
38
+ // .library/vectors.sqlite is derived, gitignored and regenerable: a reader that finds it
39
+ // broken (first-open lock contention on a fresh volume, an unopenable file) should recompute,
40
+ // not die -- the same ruling artifactFile already applies to a bad cache entry.
41
+ try { cache = emb && db.vec ? openVecCache(p) : null; }
42
+ catch (err) { process.stderr.write(`index: vector cache unavailable, recomputing: ${err.message}\n`); cache = null; }
43
+ // skipped.image is nested here, not returned as a sibling top-level `skipped`, because that
44
+ // name is already taken by the arm-level skip-reason array below (`skipped: [{arm,reason}]`,
45
+ // consumed by src/cli/index.mjs, src/cli/serve.mjs and src/mcp/tools.mjs as an iterable) --
46
+ // a same-named sibling with a different shape would break every one of those.
47
+ const embedded = { text: 0, image: 0, skipped: { image: 0 } };
34
48
  const refused = [];
35
49
  const insEntry = db.prepare(`INSERT INTO entries VALUES (@slug,@title,@tier,@source_kind,@captured_at,@dir,@body,@fm)`);
36
50
  const insTag = db.prepare(`INSERT INTO entry_tags VALUES (?,?,?)`);
@@ -58,26 +72,53 @@ export async function buildIndex({ entriesDir, indexPath, vocab, models, p }) {
58
72
  tx(e, tags);
59
73
  if (emb && db.vec) {
60
74
  const tagsStr = tags.map(([f, t]) => `${f}/${t}`).join(' ');
61
- const tv = await emb.text(e.fm.title + '\n' + searchableBody(e) + '\n' + tagsStr);
62
- insVecText.run(e.slug, Buffer.from(tv.buffer));
75
+ const text = e.fm.title + '\n' + searchableBody(e) + '\n' + tagsStr;
76
+ // Keyed on the exact string that gets embedded, so an edit that changes the text changes
77
+ // the key and re-embeds, and an edit that does not (whitespace in an unindexed field)
78
+ // correctly costs nothing.
79
+ const textKey = createHash('sha256').update(text).digest('hex');
80
+ let tv = cache?.get(`text:${emb.id}`, textKey) ?? null;
81
+ if (!tv) {
82
+ tv = Buffer.from((await emb.text(text)).buffer);
83
+ // tv is already computed; a cache write failure (SQLITE_FULL, a broken cache) must
84
+ // cost exactly the write, not the build.
85
+ try { cache?.put(`text:${emb.id}`, textKey, tv); } catch (err) {
86
+ process.stderr.write(`index: ${e.slug}: text vector cache write skipped: ${err.message}\n`);
87
+ }
88
+ embedded.text++;
89
+ }
90
+ insVecText.run(e.slug, tv);
63
91
  // An unreachable strip must not abort the whole index: skip this entry's image
64
92
  // vector and keep going, the same degradation buildUi applies to its thumbnail.
65
93
  const strip = (e.fm.captured?.captures?.[0]?.artifacts ?? []).find((a) => a?.key?.endsWith('/strip.webp'));
66
94
  if (strip) {
67
- try {
68
- const iv = await emb.image(await artifactFile(p, strip));
69
- insVecImage.run(e.slug, Buffer.from(iv.buffer));
70
- } catch (err) {
71
- process.stderr.write(`index: ${e.slug}: image vector skipped: ${err.message}\n`);
95
+ // Keyed on the digest entry.md ALREADY records, which is the whole win: on a hit there
96
+ // is no artifactFile call, so no fetch and no cache read -- a restart over an unchanged
97
+ // corpus does zero network I/O for images.
98
+ let iv = cache?.get(`image:${emb.id}`, strip.sha256) ?? null;
99
+ if (!iv) {
100
+ try {
101
+ iv = Buffer.from((await emb.image(await artifactFile(p, strip))).buffer);
102
+ cache?.put(`image:${emb.id}`, strip.sha256, iv);
103
+ embedded.image++;
104
+ } catch (err) {
105
+ iv = null;
106
+ embedded.skipped.image++;
107
+ process.stderr.write(`index: ${e.slug}: image vector skipped: ${err.message}\n`);
108
+ }
72
109
  }
110
+ if (iv) insVecImage.run(e.slug, iv);
73
111
  }
74
112
  }
75
113
  n++;
76
114
  }
77
- db.close();
115
+ // A close failure here must not mask a completed build or discard its index -- same
116
+ // reasoning as the catch block below, extended to the success path.
117
+ try { db.close(); } catch {}
118
+ try { cache?.close(); } catch {}
78
119
  fs.renameSync(tmpPath, indexPath);
79
120
  return {
80
- entries: n, refused,
121
+ entries: n, refused, embedded,
81
122
  arms: emb && db.vec ? ['fts', 'vec'] : ['fts'],
82
123
  skipped: emb
83
124
  ? (db.vec ? [] : [{ arm: 'vec', reason: 'sqlite-vec failed to load' }])
@@ -88,6 +129,7 @@ export async function buildIndex({ entriesDir, indexPath, vocab, models, p }) {
88
129
  // handle here leaks for the life of the process. A close failure must not mask
89
130
  // the original error, so it's swallowed rather than rethrown.
90
131
  try { db?.close(); } catch {}
132
+ try { cache?.close(); } catch {}
91
133
  fs.rmSync(tmpPath, { force: true });
92
134
  throw err;
93
135
  }
@@ -29,7 +29,12 @@ export async function getEmbedder({ models, mode = process.env.LIBRARY_EMBED } =
29
29
  if (mode === 'off') return null;
30
30
  if (mode === 'stub') {
31
31
  return {
32
- arm: 'vec', dims: DIMS,
32
+ // Namespaced, so no value of LIBRARY_EMBED_STUB_ID can equal the real arm's id below.
33
+ // Unprefixed, setting it to the two model constants verbatim produced a cache key
34
+ // byte-identical to a genuine real-arm build, and a later real build read the stub's
35
+ // hash-noise vectors as a warm hit — the very bug this id exists to prevent, re-entered
36
+ // through its own test seam. Reproduced before this line was added.
37
+ arm: 'vec', dims: DIMS, id: `stub:${process.env.LIBRARY_EMBED_STUB_ID ?? 'default'}`,
33
38
  text: async (s) => stubVec('t:' + s, DIMS.text),
34
39
  // ponytail: seeded on byte length + basename, not pixel content. Every
35
40
  // indexed strip shares the basename strip.webp, so two strips of the same
@@ -48,7 +53,7 @@ export async function getEmbedder({ models, mode = process.env.LIBRARY_EMBED } =
48
53
  const proc = await tf.AutoProcessor.from_pretrained(IMAGE_MODEL);
49
54
  const vision = await tf.SiglipVisionModel.from_pretrained(IMAGE_MODEL);
50
55
  return {
51
- arm: 'vec', dims: DIMS,
56
+ arm: 'vec', dims: DIMS, id: `${TEXT_MODEL}|${IMAGE_MODEL}`,
52
57
  text: async (s) => Float32Array.from((await textPipe(s, { pooling: 'mean', normalize: true })).data),
53
58
  image: async (p) => {
54
59
  const img = await tf.RawImage.read(p);
@@ -0,0 +1,53 @@
1
+ import fs from 'node:fs';
2
+ import path from 'node:path';
3
+ import { openIndex } from './query.mjs';
4
+
5
+ const SCHEMA = `
6
+ CREATE TABLE IF NOT EXISTS vectors (
7
+ kind TEXT NOT NULL,
8
+ key TEXT NOT NULL,
9
+ vec BLOB NOT NULL,
10
+ PRIMARY KEY (kind, key)
11
+ );
12
+ `;
13
+
14
+ // A separate file, deliberately. buildIndex writes index.sqlite.tmp and renames it over the real
15
+ // index on every run, so a cache table inside it would be destroyed by the very operation it is
16
+ // meant to make cheap. This one lives beside it and outlives every rebuild.
17
+ export function openVecCache(p) {
18
+ // .library/, beside index.sqlite and media/ — the one place on disk where derived bytes live,
19
+ // and the directory .gitignore covers. Loose in p.data it would be an untracked file in the
20
+ // repo root of every local checkout.
21
+ const file = path.join(p.data, '.library', 'vectors.sqlite');
22
+ fs.mkdirSync(path.dirname(file), { recursive: true });
23
+ let db;
24
+ try {
25
+ db = openIndex(file, { readonly: false });
26
+ // Unconditional, and deliberately so. A review flagged this as taking a write lock on every
27
+ // open — two processes legitimately share this volume, so that would have mattered. Measured,
28
+ // it does not: with another connection holding BEGIN IMMEDIATE, `CREATE TABLE IF NOT EXISTS`
29
+ // against an EXISTING table succeeds and takes no write lock, and only the first open, when
30
+ // the table really is absent, raises SQLITE_BUSY. Guarding this with a `SELECT name FROM
31
+ // sqlite_master` first would add a query to every open to avoid a lock that is never taken
32
+ // after the first one. Left simple on purpose.
33
+ db.exec(SCHEMA);
34
+ const sel = db.prepare('SELECT vec FROM vectors WHERE kind = ? AND key = ?');
35
+ const ins = db.prepare('INSERT OR REPLACE INTO vectors (kind, key, vec) VALUES (?, ?, ?)');
36
+ return {
37
+ // null means MISS. A row holding a zero-length blob is a hit whose value is empty, and the
38
+ // two must stay distinguishable or an empty vector re-embeds on every single start.
39
+ get(kind, key) {
40
+ const row = sel.get(kind, key);
41
+ return row ? Buffer.from(row.vec) : null;
42
+ },
43
+ put(kind, key, buf) { ins.run(kind, key, buf); },
44
+ close() { db.close(); },
45
+ };
46
+ } catch (err) {
47
+ // openVecCache can be called on every http server startup or in CLI runs on a shared
48
+ // volume; an unclosed handle here leaks for the life of the process. A close failure must
49
+ // not mask the original error, so it's swallowed rather than rethrown.
50
+ try { db?.close(); } catch {}
51
+ throw err;
52
+ }
53
+ }
@@ -0,0 +1,77 @@
1
+ import { parseEntry, writeEntry, REQUIRED_BLANKS } from '../entry.mjs';
2
+ import { resolveTerm } from '../vocab.mjs';
3
+
4
+ // Exactly the fields a session is the author of. tier and ported_from are here
5
+ // because a ported entry cannot be completed without them and the licence gate in
6
+ // validateEntry still judges the result. Everything absent from this list is
7
+ // provenance the server measured, and is refused by name below.
8
+ const ALLOWED = new Set([...REQUIRED_BLANKS, 'tier', 'ported_from']);
9
+
10
+ // Which vocabulary facet governs which field. A field absent here takes free text.
11
+ const FACET_OF = { roles: 'role', feel: 'feel', 'motion.devices': 'motion', tier: 'tier' };
12
+
13
+ function get(obj, dotted) {
14
+ return dotted.split('.').reduce((o, k) => (o == null ? undefined : o[k]), obj);
15
+ }
16
+ function set(obj, dotted, value) {
17
+ const keys = dotted.split('.');
18
+ const last = keys.pop();
19
+ let o = obj;
20
+ for (const k of keys) {
21
+ if (o[k] == null || typeof o[k] !== 'object') o[k] = {};
22
+ o = o[k];
23
+ }
24
+ o[last] = value;
25
+ }
26
+ function isBlank(v) {
27
+ return v == null || v === '' || (Array.isArray(v) && v.length === 0);
28
+ }
29
+
30
+ export function setFields({ dir, fields, vocab }) {
31
+ const entry = parseEntry(dir);
32
+ const fm = entry.fm;
33
+ let body = entry.body;
34
+ const refused = [];
35
+ const staged = [];
36
+
37
+ for (const [key, value] of Object.entries(fields ?? {})) {
38
+ if (!ALLOWED.has(key)) {
39
+ refused.push(`${key}: not a judgement field — set_fields writes ${[...ALLOWED].join(', ')} and nothing else`);
40
+ continue;
41
+ }
42
+ const facet = FACET_OF[key];
43
+ if (facet) {
44
+ const terms = Array.isArray(value) ? value : [value];
45
+ const canon = [];
46
+ for (const t of terms) {
47
+ const c = resolveTerm(vocab, facet, t);
48
+ if (c === null) refused.push(`${key}: unknown ${facet} term "${t}"`);
49
+ else canon.push(c);
50
+ }
51
+ staged.push([key, Array.isArray(value) ? canon : canon[0]]);
52
+ continue;
53
+ }
54
+ staged.push([key, value]);
55
+ }
56
+
57
+ // Nothing is written when anything was refused. A partial write leaves the agent
58
+ // reasoning about a draft that is neither what it sent nor what it had.
59
+ if (refused.length) {
60
+ return { ok: false, slug: fm.slug, blanks: REQUIRED_BLANKS.filter((k) => isBlank(get({ ...fm, body }, k))), written: [], refused };
61
+ }
62
+
63
+ for (const [key, value] of staged) {
64
+ if (key === 'body') body = value;
65
+ else set(fm, key, value);
66
+ }
67
+ writeEntry(dir, fm, body);
68
+
69
+ const view = { ...fm, body };
70
+ return {
71
+ ok: true,
72
+ slug: fm.slug,
73
+ blanks: REQUIRED_BLANKS.filter((k) => isBlank(get(view, k))),
74
+ written: staged.map(([k]) => k),
75
+ refused: [],
76
+ };
77
+ }
@@ -40,8 +40,14 @@ session did not make.
40
40
  \`frames: 36\` before tagging. The server caps at 36 frames; beyond about 18 seconds a
41
41
  section can still be missed, so build a denser sheet yourself from the source file
42
42
  before writing the motion note.
43
- 4. **Tag.** Fill every field in the returned \`blanks\` list in \`<dir>/entry.md\`, per the
44
- rubric below. Write the motion note last, after the roster.
43
+ 4. **Tag.** Fill every field in the returned \`blanks\` list, per the rubric below. Write
44
+ the motion note last, after the roster. Two ways, same result: call \`set_fields\` with
45
+ those field names — this needs **no filesystem** and is the only way that works against
46
+ a hosted server — or, in a local checkout, edit \`<dir>/entry.md\` directly.
47
+ \`set_fields\` resolves aliases and refuses an unknown term by name without writing
48
+ anything, so a refusal never leaves a half-filled draft. It writes judgement fields
49
+ only; provenance is refused. \`get_vocab\` returns the terms if you did not read them
50
+ above.
45
51
  5. **Approve.** Show the draft to the user and **show the draft and wait for a yes** before
46
52
  anything else. Apply their edits.
47
53
  6. **Save.** Call \`save_entry\`. On \`refused\`, show the field names, fix, and call again;
package/src/mcp/tools.mjs CHANGED
@@ -213,6 +213,15 @@ export function makeTools(ctx) {
213
213
  return { arms: arms ?? ['fts'], ...(skipped ? { skipped } : {}), feel, roles };
214
214
  }),
215
215
  },
216
+ {
217
+ name: 'get_vocab',
218
+ description: 'The five controlled facets and the alias map, read from vocab.yaml. roles and feel tag an entry, motion names the data-motion devices, source and tier gate the licence. A term is added by PR, never by an ingest, so an unknown term is a refusal rather than a new term.',
219
+ schema: z.object({}),
220
+ handler: async () => ({
221
+ ...Object.fromEntries(Object.entries(ctx.vocab.facets).map(([k, v]) => [k, [...v]])),
222
+ aliases: Object.fromEntries(ctx.vocab.aliases),
223
+ }),
224
+ },
216
225
  {
217
226
  name: 'add',
218
227
  description: 'Ingest a video/animated image, or measure a live https:// URL with the system Chrome: sample frames or measure into strip.webp/measure.json and write a draft entry. Returns the blank fields the caller must fill before save_entry. Pass attach to add a second capture of the same site — a video beside a live URL — instead of refusing; the existing entry\'s tags, body, palette and type are left untouched.',
@@ -238,6 +247,26 @@ export function makeTools(ctx) {
238
247
  return (await import('../cli/add.mjs')).addEntry(a);
239
248
  },
240
249
  },
250
+ {
251
+ name: 'add_batch',
252
+ description: "Ingest every media file in the server's inbox in one run: one draft per file, an existing slug skipped so an interrupted run resumes, and a failed file costing only itself. Returns one record per file. Takes no path — the walk is the inbox and nothing else. Each drafted entry still needs set_fields then save_entry.",
253
+ schema: z.object({
254
+ source: z.object({
255
+ kind: z.enum(['own', 'open', 'paid', 'public-site']),
256
+ url: z.string().url().optional(),
257
+ license: z.string().min(1),
258
+ }).optional(),
259
+ }),
260
+ handler: async ({ source }) => {
261
+ const { addInbox } = await import('../cli/add.mjs');
262
+ const results = await addInbox({ source });
263
+ return {
264
+ results,
265
+ drafted: results.filter((r) => r.status === 'drafted').length,
266
+ refused: results.filter((r) => r.status === 'refused').length,
267
+ };
268
+ },
269
+ },
241
270
  {
242
271
  name: 'refresh',
243
272
  description: 'Re-measure a drafted URL entry (source.url set, captured.method measured|both) with the system Chrome: rewrites measure.json, bumps captured.at, and reindexes. Refuses an entry with no URL or one drafted from frames only.',
@@ -253,6 +282,18 @@ export function makeTools(ctx) {
253
282
  return saveEntry({ dir: path.join(ctx.paths.entries, slug), vocab: ctx.vocab, indexPath: ctx.paths.index, entriesDir: ctx.paths.entries, models: ctx.paths.models, p: ctx.paths });
254
283
  },
255
284
  },
285
+ {
286
+ name: 'set_fields',
287
+ description: "Fill a draft's judgement fields: roles, feel, motion.devices, motion.notes, palette, type, body, tier, ported_from — never provenance. Vocabulary terms are resolved through aliases and an unknown one is refused by name with nothing written, so a refusal never leaves a half-filled draft. Returns the fields still blank. This is the step between add and save_entry, and it is the only one that does not need a filesystem.",
288
+ schema: z.object({
289
+ slug: z.string().regex(/^[a-z0-9-]+$/),
290
+ fields: z.record(z.any()),
291
+ }),
292
+ handler: async ({ slug, fields }) => {
293
+ const { setFields } = await import('../ingest/fields.mjs');
294
+ return setFields({ dir: path.join(ctx.paths.entries, slug), fields, vocab: ctx.vocab });
295
+ },
296
+ },
256
297
  ];
257
298
  }
258
299
 
package/src/r2.mjs CHANGED
@@ -59,15 +59,50 @@ export function sha256(body) {
59
59
  return createHash('sha256').update(body).digest('hex');
60
60
  }
61
61
 
62
+ // Extracted from s3Upload, which built this inline. Two constructions of the same client that can
63
+ // drift apart is exactly the kind of duplication that makes one of them quietly wrong later.
64
+ function clientConfig(config) {
65
+ return { region: 'auto', endpoint: `https://${config.account}.r2.cloudflarestorage.com`,
66
+ credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey } };
67
+ }
68
+
62
69
  export async function s3Upload(config) {
63
70
  const { S3Client, PutObjectCommand } = await import('@aws-sdk/client-s3');
64
- const client = new S3Client({ region: 'auto', endpoint: `https://${config.account}.r2.cloudflarestorage.com`,
65
- credentials: { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey } });
71
+ const client = new S3Client(clientConfig(config));
66
72
  return (args) => client.send(new PutObjectCommand(args));
67
73
  }
68
74
 
69
- // Upload, then prove the object is publicly readable and byte-identical before any caller records
70
- // the key. motion.mjs established this order: an entry must never name an object nobody can read.
75
+ // Mirrors s3Upload: the SDK is imported lazily so a reader never loads it.
76
+ export async function s3Head(config) {
77
+ const { S3Client, HeadObjectCommand } = await import('@aws-sdk/client-s3');
78
+ const client = new S3Client(clientConfig(config));
79
+ return async ({ Bucket, Key }) => {
80
+ const r = await client.send(new HeadObjectCommand({ Bucket, Key }));
81
+ return { Metadata: r.Metadata ?? {} };
82
+ };
83
+ }
84
+
85
+ const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
86
+
87
+ // Only the PUT is retried. The range-fetch verification below is not: a verification failure
88
+ // means the object in the bucket is not what we sent, and repeating the read cannot change that
89
+ // -- it would just delay a refusal that is already correct.
90
+ async function withRetry(fn, retries, sleep) {
91
+ for (let attempt = 1; ; attempt++) {
92
+ try { return await fn(); }
93
+ catch (err) {
94
+ if (attempt >= retries) throw err;
95
+ await sleep(250 * 2 ** (attempt - 1));
96
+ }
97
+ }
98
+ }
99
+
100
+ // Upload -- or, when the digest already matches what's in the bucket, skip the PUT -- then prove
101
+ // the object is publicly readable and byte-identical before any caller records the key. motion.mjs
102
+ // established this order: an entry must never name an object nobody can read, and that holds for
103
+ // the skip path too. The HEAD above is an authenticated S3-API call against the origin; it proves
104
+ // the bytes are right, not that the public edge serves them, so the skip still pays for a 32-byte
105
+ // range-fetch through R2_PUBLIC_BASE_URL -- cheap next to the full upload it avoided.
71
106
  //
72
107
  // `immutable` is NOT carried over from motion.mjs with it. That key embeds the digest
73
108
  // (motion/<slug>/<id>-<sha256[0:16]>.mp4) so it never names two different bodies; an artifact key
@@ -78,18 +113,38 @@ export async function s3Upload(config) {
78
113
  // verification below, which range-fetches through R2_PUBLIC_BASE_URL (the edge): under `immutable`
79
114
  // it can compare the new body against a cached old object and throw `public bytes differ` AFTER
80
115
  // the object was overwritten. The cost is a re-capture taking up to max-age to reach the edge.
81
- export async function putAndVerify({ config, key, body, contentType, upload, fetcher = fetch }) {
82
- const digest = sha256(body);
83
- await upload({ Bucket: config.bucket, Key: key, Body: body, ContentType: contentType,
84
- CacheControl: 'public, max-age=300', Metadata: { sha256: digest } });
116
+ // The one range-fetch check, run after either a real upload or a skip. Two copies of this would
117
+ // drift -- the whole point of the invariant above is that neither path is allowed to skip it.
118
+ async function verifyPublic({ config, key, body, fetcher, digest, uploaded }) {
85
119
  const url = objectUrl(config, key);
86
120
  const response = await fetcher(url, { headers: { Range: 'bytes=0-31' }, signal: AbortSignal.timeout(20000) });
87
121
  const range = response.headers.get('content-range');
88
122
  if (response.status !== 206 || range !== `bytes 0-31/${body.length}`) {
89
123
  await response.body?.cancel();
90
- throw new Error(`uploaded ${key}, but public verification failed (HTTP ${response.status}); nothing was recorded`);
124
+ const verb = uploaded ? 'uploaded' : 'matched';
125
+ throw new Error(`${verb} ${key}, but public verification failed (HTTP ${response.status}); nothing was recorded`);
126
+ }
127
+ const headBytes = Buffer.from(await response.arrayBuffer());
128
+ if (!headBytes.equals(body.subarray(0, 32))) throw new Error(`public bytes differ for ${key}`);
129
+ return { key, bytes: body.length, sha256: digest, uploaded };
130
+ }
131
+
132
+ export async function putAndVerify({ config, key, body, contentType, upload, head, fetcher = fetch,
133
+ retries = 3, sleep = defaultSleep }) {
134
+ const digest = sha256(body);
135
+ // Skip a PUT whose bytes are already in the bucket. The digest is compared against the object's
136
+ // own recorded metadata, not against its length: two different objects of the same size would
137
+ // otherwise read as identical, which is the exact mistake the artifact cache made and that
138
+ // 1.0.1 had to fix. An object with NO recorded digest is re-uploaded rather than assumed
139
+ // identical -- anything written before this field existed has none, and guessing would skip a
140
+ // needed write forever.
141
+ if (head) {
142
+ try {
143
+ const existing = await head({ Bucket: config.bucket, Key: key });
144
+ if (existing?.Metadata?.sha256 === digest) return verifyPublic({ config, key, body, fetcher, digest, uploaded: false });
145
+ } catch { /* NotFound, or a HEAD this account cannot make: fall through and upload */ }
91
146
  }
92
- const head = Buffer.from(await response.arrayBuffer());
93
- if (!head.equals(body.subarray(0, 32))) throw new Error(`public bytes differ for ${key}`);
94
- return { key, bytes: body.length, sha256: digest };
147
+ await withRetry(() => upload({ Bucket: config.bucket, Key: key, Body: body, ContentType: contentType,
148
+ CacheControl: 'public, max-age=300', Metadata: { sha256: digest } }), retries, sleep);
149
+ return verifyPublic({ config, key, body, fetcher, digest, uploaded: true });
95
150
  }
package/src/ui/app.js CHANGED
@@ -1,5 +1,8 @@
1
1
  // Vanilla, no build step: filters the manifest already rendered into `.card`
2
2
  // data attributes. Within one facet, checked boxes OR together; across facets, AND.
3
+ // Filtering sets eligibility only (dataset.filtered); repage(), defined globally by the inline
4
+ // script in index.html loaded before this file, is what decides what's actually painted. Paging
5
+ // stays a view over whatever the filter left eligible, never a filter of its own.
3
6
  (function () {
4
7
  var grid = document.getElementById('grid');
5
8
  var cards = grid ? Array.prototype.slice.call(grid.querySelectorAll('.card')) : [];
@@ -24,8 +27,9 @@
24
27
  var have = (card.dataset[facets[i]] || '').split(' ');
25
28
  ok = want.some(function (t) { return have.indexOf(t) !== -1; });
26
29
  }
27
- card.hidden = !ok;
30
+ card.dataset.filtered = ok ? '' : 'out';
28
31
  });
32
+ repage();
29
33
  }
30
34
 
31
35
  var inputs = document.querySelectorAll('#filters input[type="checkbox"]');
package/src/ui/build.mjs CHANGED
@@ -46,7 +46,7 @@ function filterFieldset(facet, terms) {
46
46
  }
47
47
 
48
48
  function card(row) {
49
- const img = row.thumbnail ? `<img src="${row.thumbnail}" alt="" loading="lazy" decoding="async">` : `<div class="ph"></div>`;
49
+ const img = row.thumbnail ? `<img src="${row.thumbnail}" alt="" loading="lazy" decoding="async" width="640" height="480">` : `<div class="ph"></div>`;
50
50
  const badge = row.draft ? ' <span class="badge">draft</span>' : '';
51
51
  const tags = esc([...row.roles, ...row.feel, ...row.motion].join(', '));
52
52
  const data = `data-role="${esc(row.roles.join(' '))}" data-feel="${esc(row.feel.join(' '))}" data-motion="${esc(row.motion.join(' '))}"`;
@@ -150,13 +150,20 @@ function writeEntryPage(outDir, entry, row, previewMediaDir, measureFile) {
150
150
  export async function buildUi({ indexPath, entriesDir, outDir, previewMediaDir, p }) {
151
151
  void indexPath;
152
152
  const vocab = loadVocab();
153
- fs.rmSync(outDir, { recursive: true, force: true });
153
+ // Wipe the HTML, keep the pixels. Re-encoding 2000 unchanged thumbnails on every deploy is the
154
+ // single most expensive thing this build does, and a thumbnail is a pure function of the strip
155
+ // digest already recorded in entry.md — so it is safe to keep and cheap to invalidate.
156
+ for (const name of fs.existsSync(outDir) ? fs.readdirSync(outDir) : []) {
157
+ if (name === 'thumbnails' || name === 'strips') continue;
158
+ fs.rmSync(path.join(outDir, name), { recursive: true, force: true });
159
+ }
154
160
  fs.mkdirSync(path.join(outDir, 'entry'), { recursive: true });
155
161
  fs.mkdirSync(path.join(outDir, 'strips'), { recursive: true });
156
162
  fs.mkdirSync(path.join(outDir, 'thumbnails'), { recursive: true });
157
163
 
158
164
  const rows = [];
159
165
  const unresolved = [];
166
+ let rendered = 0;
160
167
  for (const dir of listEntries(entriesDir)) {
161
168
  let entry;
162
169
  try { entry = parseEntry(dir); } catch (err) { process.stderr.write(`ui: skipping ${path.basename(dir)}: ${err.message}\n`); continue; }
@@ -168,24 +175,45 @@ export async function buildUi({ indexPath, entriesDir, outDir, previewMediaDir,
168
175
  const slug = path.basename(dir);
169
176
  let stripSrc = null;
170
177
  const strip = (entry.fm.captured?.captures?.[0]?.artifacts ?? []).find((a) => a?.key?.endsWith('/strip.webp'));
178
+ // The index is a browsing surface. Its previews must not make each visitor
179
+ // download a multi-frame source sheet; the full strip remains on the detail page.
180
+ const thumb = path.join(outDir, 'thumbnails', `${slug}.webp`);
181
+ const stamp = path.join(outDir, 'thumbnails', `${slug}.sha256`);
182
+ const stripOut = path.join(outDir, 'strips', `${slug}.webp`);
183
+ // The sidecar records WHICH strip produced this thumbnail, AND which encoder made it: a strip
184
+ // whose digest is unchanged still needs a new render once the encode below changes shape, or
185
+ // a deploy that ships a new encoder serves the old pixels forever. THUMB sits next to the
186
+ // sharp() call it describes (below) so the two cannot drift apart silently -- bump it when
187
+ // that call's parameters change.
188
+ const THUMB = '640x480-cover-north-webp-q72-e4';
189
+ // Consulted BEFORE artifactFile, not after: the selective wipe above preserves thumbnails/
190
+ // and strips/, so when the stamp matches and both outputs are already on disk there is
191
+ // nothing to resolve -- no cache read, no hash, no fetch, for this entry's strip.
192
+ let current = null;
193
+ try { current = fs.readFileSync(stamp, 'utf8').trim(); } catch {}
194
+ const sidecarHit = !!strip && current === `${THUMB}:${strip.sha256}` && fs.existsSync(thumb) && fs.existsSync(stripOut);
195
+ // Same treatment for measure.json: a record to resolve, never a file next to entry.md. It has
196
+ // no sidecar of its own and is a different artifact, so it always resolves here regardless of
197
+ // the strip's sidecar outcome.
198
+ let measureSrc = null;
199
+ const measured = (entry.fm.captured?.captures?.[0]?.artifacts ?? []).find((a) => a?.key?.endsWith('/measure.json'));
200
+ if (measured) { try { measureSrc = await artifactFile(p, measured); } catch (err) { measureSrc = null; unresolved.push(err); } }
171
201
  // One unreachable object must not take the gallery down with it: the entry renders without a
172
202
  // thumbnail, exactly as a draft does, and the build continues.
173
203
  // Degrading silently is what made a misconfigured host indistinguishable from a broken one:
174
204
  // a gallery of 31 grey boxes, `ok: true` from healthz, and not one line saying why. Swallow
175
205
  // the failure, still — but count it and keep the first reason.
176
- if (strip) { try { stripSrc = await artifactFile(p, strip); } catch (err) { stripSrc = null; unresolved.push(err); } }
177
- // Same treatment for measure.json: a record to resolve, never a file next to entry.md.
178
- let measureSrc = null;
179
- const measured = (entry.fm.captured?.captures?.[0]?.artifacts ?? []).find((a) => a?.key?.endsWith('/measure.json'));
180
- if (measured) { try { measureSrc = await artifactFile(p, measured); } catch (err) { measureSrc = null; unresolved.push(err); } }
181
- const hasStrip = stripSrc !== null;
182
- if (hasStrip) {
183
- fs.copyFileSync(stripSrc, path.join(outDir, 'strips', `${slug}.webp`));
184
- // The index is a browsing surface. Its previews must not make each visitor
185
- // download a multi-frame source sheet; the full strip remains on the detail page.
186
- await sharp(stripSrc).resize(640, 480, { fit: 'cover', position: 'north' })
187
- .webp({ quality: 72, effort: 4 }).toFile(path.join(outDir, 'thumbnails', `${slug}.webp`));
206
+ if (strip && !sidecarHit) {
207
+ try {
208
+ stripSrc = await artifactFile(p, strip);
209
+ fs.copyFileSync(stripSrc, stripOut);
210
+ await sharp(stripSrc).resize(640, 480, { fit: 'cover', position: 'north' }) // THUMB above must name these params
211
+ .webp({ quality: 72, effort: 4 }).toFile(thumb);
212
+ fs.writeFileSync(stamp, `${THUMB}:${strip.sha256}`);
213
+ rendered++;
214
+ } catch (err) { stripSrc = null; unresolved.push(err); }
188
215
  }
216
+ const hasStrip = sidecarHit || stripSrc !== null;
189
217
  const fm = entry.fm;
190
218
  const capturedAt = fm.captured?.at;
191
219
  const captured = capturedAt instanceof Date ? capturedAt.toISOString().slice(0, 10) : String(capturedAt ?? '');
@@ -222,5 +250,5 @@ export async function buildUi({ indexPath, entriesDir, outDir, previewMediaDir,
222
250
  if (unresolved.length) {
223
251
  process.stderr.write(`ui: ${unresolved.length} artifact(s) could not be resolved, so those entries render without a thumbnail; first reason: ${unresolved[0].message}\n`);
224
252
  }
225
- return { entries: rows.length - drafts, drafts, unresolved: unresolved.length };
253
+ return { entries: rows.length - drafts, drafts, unresolved: unresolved.length, rendered };
226
254
  }
@@ -26,9 +26,32 @@
26
26
  <label><input type="checkbox" id="drafts-toggle"> Show drafts</label>
27
27
  </form>
28
28
  {{NOTICE}}
29
- <div id="grid">
29
+ <div id="grid" data-page-size="60">
30
30
  {{GRID}}
31
31
  </div>
32
+ <button id="show-more" hidden>Show more</button>
33
+ <script>
34
+ // Paging is a view over whatever the filter left eligible, never a filter itself: `cards` is
35
+ // still every card in the document, so a search always sees the whole corpus. Only how many
36
+ // eligible cards are currently painted changes.
37
+ var cards = document.querySelectorAll(".card");
38
+ var PAGE = parseInt(document.querySelector("[data-page-size]").dataset.pageSize, 10);
39
+ var shown = PAGE;
40
+ function repage() {
41
+ var n = 0;
42
+ cards.forEach(function (c) {
43
+ if (c.dataset.filtered === "out") { c.hidden = true; return; }
44
+ n++;
45
+ c.hidden = n > shown;
46
+ });
47
+ var more = document.getElementById("show-more");
48
+ more.hidden = n <= shown;
49
+ }
50
+ document.getElementById("show-more").addEventListener("click", function () {
51
+ shown += PAGE;
52
+ repage();
53
+ });
54
+ </script>
32
55
  <script src="app.js"></script>
33
56
  </body>
34
57
  </html>