create-evda 0.1.9 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/presets.mjs CHANGED
@@ -28,7 +28,10 @@
28
28
  // that are missing, and reports the rest: a manifest that already says the same thing is skipped,
29
29
  // one that says something else is a conflict and is left exactly as it is, and a preset path that
30
30
  // already exists as a file is a conflict. Nothing is moved, renamed or deleted, so applying twice
31
- // writes nothing the second time. A shipped preset that does not validate fails
31
+ // writes nothing the second time. A folder typed person also brings its layout (HQ-2024): a sibling
32
+ // Processing/ for processing docs, and a README inside the folder stating the one-file-per-person
33
+ // rule doctor checks, each only when missing and only when the folder's manifest is the preset's.
34
+ // A shipped preset that does not validate fails
32
35
  // test/presets.test.mjs, so the person running it never meets one.
33
36
 
34
37
  import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
@@ -40,6 +43,7 @@ import { validateFolderManifest, FOLDER_MANIFEST, FOLDER_TYPES } from "./folders
40
43
  import { ROUTINE_KEYS } from "./routines.mjs";
41
44
  import { onlyKeys, proposeCircle, refuse, PROPOSALS } from "./tools.mjs";
42
45
  import { EvdaError } from "./errors.mjs";
46
+ import { PERSON_FOLDER_README, PROCESSING_DIR, README_NAME } from "./person-folders.mjs";
43
47
 
44
48
  export const PRESETS_DIR = resolve(dirname(fileURLToPath(import.meta.url)), "..", "presets");
45
49
  export const PRESET_FILE = "preset.json";
@@ -223,6 +227,7 @@ const viewHeader = (v) => ["---", `view: ${v.view}`, `of: ${v.of}`, `in: ${v.in}
223
227
  // create directories that do not exist yet
224
228
  // write manifests that do not exist yet ({ path, folder, manifest })
225
229
  // views view files a manifest declares that do not exist yet
230
+ // files files a person folder brings that do not exist yet ({ path, text }): its README
226
231
  // matched manifests that exist and already say what the preset says (skipped)
227
232
  // conflicts manifests that exist and say something else, and preset paths that are files
228
233
  // ({ path, why }); reported, never overwritten
@@ -231,7 +236,7 @@ const viewHeader = (v) => ["---", `view: ${v.view}`, `of: ${v.of}`, `in: ${v.in}
231
236
  export function applyPreset(root, id, { dry = false, now = new Date(), dir = PRESETS_DIR } = {}) {
232
237
  const evda = openEvda(resolve(root));
233
238
  const preset = loadPreset(id, { dir });
234
- const plan = { id: preset.id, title: preset.title, root: evda.dir, dry, create: [], write: [], views: [], matched: [], conflicts: [], circles: [], proposed: [], routines: preset.routines ?? [], folders: preset.folders.length, roles: [] };
239
+ const plan = { id: preset.id, title: preset.title, root: evda.dir, dry, create: [], write: [], files: [], views: [], matched: [], conflicts: [], circles: [], proposed: [], routines: preset.routines ?? [], folders: preset.folders.length, roles: [] };
235
240
  // The four roles the record does not declare yet. A key it has with another path is a conflict,
236
241
  // left as it is; a path it already maps under a key of the same role is already there.
237
242
  const has = evda.manifest.mapping;
@@ -263,15 +268,29 @@ export function applyPreset(root, id, { dry = false, now = new Date(), dir = PRE
263
268
  const want = manifestFor(f);
264
269
  const mpath = `${f.path}/${FOLDER_MANIFEST}`;
265
270
  const mabs = join(abs, FOLDER_MANIFEST);
271
+ let ours = true;
266
272
  if (!existsSync(mabs)) plan.write.push({ path: mpath, folder: f, manifest: want });
267
- else if (!lstatSync(mabs).isFile()) plan.conflicts.push({ path: mpath, why: "exists and is not a file" });
273
+ else if (!lstatSync(mabs).isFile()) { plan.conflicts.push({ path: mpath, why: "exists and is not a file" }); ours = false; }
268
274
  else {
269
275
  let existing;
270
276
  try { existing = JSON.parse(readFileSync(mabs, "utf8")); } catch { plan.conflicts.push({ path: mpath, why: "exists and is not valid JSON" }); continue; }
271
277
  const diff = differences(existing, want);
272
- if (diff.length) plan.conflicts.push({ path: mpath, why: `exists and differs: ${diff.join("; ")}` });
278
+ if (diff.length) { plan.conflicts.push({ path: mpath, why: `exists and differs: ${diff.join("; ")}` }); ours = false; }
273
279
  else plan.matched.push({ path: mpath });
274
280
  }
281
+ if (f.type === "person" && ours) {
282
+ const readme = `${f.path}/${README_NAME}`;
283
+ const rabs = join(abs, README_NAME);
284
+ if (!existsSync(rabs)) plan.files.push({ path: readme, text: PERSON_FOLDER_README });
285
+ else if (!lstatSync(rabs).isFile()) plan.conflicts.push({ path: readme, why: "exists and is not a file" });
286
+ const parent = f.path.includes("/") ? f.path.slice(0, f.path.lastIndexOf("/")) : "";
287
+ const processing = parent ? `${parent}/${PROCESSING_DIR}` : PROCESSING_DIR;
288
+ const pfile = blockedBy(evda.dir, processing);
289
+ if (pfile) {
290
+ if (!blocked.has(pfile)) plan.conflicts.push({ path: pfile, why: `exists as a file, so ${processing}/ for processing docs beside ${f.path} cannot be made there` });
291
+ blocked.add(pfile);
292
+ } else if (!existsSync(join(evda.dir, processing)) && !plan.create.includes(processing)) plan.create.push(processing);
293
+ }
275
294
  for (const v of want.views) {
276
295
  if (!existsSync(resolve(evda.dir, v.path))) plan.views.push({ path: v.path, view: v });
277
296
  }
@@ -295,6 +314,7 @@ export function applyPreset(root, id, { dry = false, now = new Date(), dir = PRE
295
314
  }
296
315
  for (const rel of plan.create) mkdirSync(join(evda.dir, rel), { recursive: true });
297
316
  for (const w of plan.write) writeFileSync(join(evda.dir, w.path), JSON.stringify(w.manifest, null, 2) + "\n", { flag: "wx" });
317
+ for (const w of plan.files) writeFileSync(join(evda.dir, w.path), w.text, { flag: "wx" });
298
318
  for (const v of plan.views) {
299
319
  mkdirSync(dirname(resolve(evda.dir, v.path)), { recursive: true });
300
320
  writeFileSync(resolve(evda.dir, v.path), viewHeader(v.view), { flag: "wx" });
@@ -317,7 +337,7 @@ export function diffPreset(root, id, opts = {}) {
317
337
  return applyPreset(root, id, { ...opts, dry: true });
318
338
  }
319
339
 
320
- export const planIsEmpty = (plan) => !plan.create.length && !plan.write.length && !plan.views.length && !plan.circles.length && !plan.conflicts.length && !(plan.roles ?? []).length;
340
+ export const planIsEmpty = (plan) => !plan.create.length && !plan.write.length && !(plan.files ?? []).length && !plan.views.length && !plan.circles.length && !plan.conflicts.length && !(plan.roles ?? []).length;
321
341
 
322
342
  // The plan in plain words, for the owner to say yes or no to. Names each folder by its note.
323
343
  export function summary(plan, preset = loadPreset(plan.id)) {
@@ -344,6 +364,7 @@ export function planLines(plan, { diff = false } = {}) {
344
364
  const verb = plan.dry || diff ? "would" : "did";
345
365
  for (const p of plan.create) out.push(` ${verb === "did" ? "created " : "create "} ${p}/`);
346
366
  for (const w of plan.write) out.push(` ${verb === "did" ? "wrote " : "write "} ${w.path} type ${w.manifest.type}${w.manifest.connectors.length ? `, connectors ${w.manifest.connectors.join(", ")}` : ""}`);
367
+ for (const w of plan.files ?? []) out.push(` ${verb === "did" ? "wrote " : "write "} ${w.path} the person folder's rule`);
347
368
  for (const r of plan.roles ?? []) out.push(` ${verb === "did" ? "declared" : "declare "} ${r.key} -> ${r.rel} in evda.json`);
348
369
  for (const v of plan.views) out.push(` ${verb === "did" ? "wrote " : "write "} ${v.path} view ${v.view.view} (header only)`);
349
370
  if (!diff) for (const m of plan.matched) out.push(` skipped ${m.path} already says the same`);
package/lib/publish.mjs CHANGED
@@ -29,7 +29,9 @@
29
29
  // written, and a hit refuses. A label is its opaque id, its namespace, its own directory, a
30
30
  // member path, or its name between slashes. The exported files' own paths are the released
31
31
  // set (doctor prints them every run) and are left out of that search, since a marked file
32
- // necessarily lives under a keyed circle's member path.
32
+ // necessarily lives under a keyed circle's member path. A markdown file's `sources:`
33
+ // frontmatter field is cut from its exported bytes before this search (HQ-2698): it is
34
+ // provenance, the record's own paths, and is never published.
33
35
  //
34
36
  // The fourth check in the spec (the site holds no credential to anything else) is a property of
35
37
  // where the export is served, not of this code.
@@ -70,6 +72,41 @@ function filesUnder(abs) {
70
72
  const posix = (rel) => rel.split(sep).join("/");
71
73
  const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
72
74
 
75
+ // Provenance (HQ-2698). A top-level `sources:` field in a markdown file's frontmatter lists the
76
+ // record paths the file was written from: a blog Companion names the private draft and the journal
77
+ // days behind it. That is the record's own bookkeeping, never part of what a stranger reads, so it
78
+ // is cut from every exported markdown file, whatever folder or rule released it, the way a sealed
79
+ // section is. Only the frontmatter field is cut: a private path anywhere in the body still reaches
80
+ // the stranger check and refuses. The value runs to the next top-level key, so a block list,
81
+ // indented or not, a flow list on one or more lines and a bare string all go with the key.
82
+ const PROVENANCE_KEY = /^(["']?)sources\1[ \t]*:/;
83
+ const isFence = (line) => line.replace(/\r$/, "") === "---";
84
+ const continues = (line) => /^[ \t]/.test(line) || /^-([ \t]|\r?$)/.test(line);
85
+ const blank = (line) => line.replace(/\r$/, "").trim() === "";
86
+ export function withoutProvenance(text) {
87
+ const lines = String(text).split("\n");
88
+ if (!isFence(lines[0])) return { text, cut: 0 };
89
+ let end = 1;
90
+ while (end < lines.length && !isFence(lines[end])) end++;
91
+ if (end >= lines.length) return { text, cut: 0 };
92
+ const kept = [lines[0]];
93
+ let cut = 0;
94
+ for (let i = 1; i < end; i++) {
95
+ if (!PROVENANCE_KEY.test(lines[i])) { kept.push(lines[i]); continue; }
96
+ cut++;
97
+ let j = i + 1;
98
+ for (;;) {
99
+ let k = j;
100
+ while (k < end && blank(lines[k])) k++;
101
+ if (k < end && continues(lines[k])) j = k + 1;
102
+ else break;
103
+ }
104
+ i = j - 1;
105
+ }
106
+ if (!cut) return { text, cut: 0 };
107
+ return { text: [...kept, ...lines.slice(end)].join("\n"), cut };
108
+ }
109
+
73
110
  // Build the export in memory and run the checks. Writes nothing.
74
111
  export function buildPublicExport(evda, { circleName = null, publishedAt = new Date() } = {}) {
75
112
  const circle = publicCircle(evda, circleName);
@@ -114,14 +151,21 @@ export function buildPublicExport(evda, { circleName = null, publishedAt = new D
114
151
  // A sealed section (HQ-1220) is cut out of every markdown file on the way out; the rest of the
115
152
  // file is exported, and nothing marks where the section was. A sealed FILE never gets here:
116
153
  // released() refuses it, and a refusal refuses the publish above.
154
+ // A file's `sources:` provenance is cut the same way (withoutProvenance above, HQ-2698).
117
155
  let sealedCut = 0;
156
+ let provenanceCut = 0;
118
157
  const exportBytes = (abs) => {
119
158
  const bytes = readFileSync(abs);
120
- if (!/\.md$/i.test(abs) || !carries("publish", "sealed")) return bytes;
121
- const cut = withoutSealed(bytes.toString("utf8"));
122
- if (!cut.sections) return bytes;
123
- sealedCut += cut.sections;
124
- return Buffer.from(cut.text, "utf8");
159
+ if (!/\.md$/i.test(abs)) return bytes;
160
+ let text = bytes.toString("utf8");
161
+ let changed = false;
162
+ if (carries("publish", "sealed")) {
163
+ const cut = withoutSealed(text);
164
+ if (cut.sections) { sealedCut += cut.sections; text = cut.text; changed = true; }
165
+ }
166
+ const prov = withoutProvenance(text);
167
+ if (prov.cut) { provenanceCut += 1; text = prov.text; changed = true; }
168
+ return changed ? Buffer.from(text, "utf8") : bytes;
125
169
  };
126
170
  const loaded = files.filter((f) => !consumed.has(f.path)).map((f) => ({ path: f.path, bytes: exportBytes(f.abs), by: f.by }));
127
171
  for (const v of rendered.views) loaded.push({ path: v.path, bytes: v.bytes, by: "view", view: v.view, sources: v.entries.map((e) => ({ path: e.path, sha256: e.sha256 })) });
@@ -157,9 +201,9 @@ export function buildPublicExport(evda, { circleName = null, publishedAt = new D
157
201
  // What identifies a private circle to a stranger is a LABEL: a path segment, the circle's own
158
202
  // directory, or the opaque id that travels on a Change and in an export manifest. A name
159
203
  // matched as a word anywhere in the bytes is not that test, and on a real record it refuses
160
- // every publish there is: his private circles are core, health, money, people, school,
161
- // therapy, wellness and workshop, and his public fact sheet says "10 core mechanics" and
162
- // "help other people understand me" (2026-09-21, HQ-1199). So the name is matched only
204
+ // every publish there is: private circles are named with ordinary words (a circle called
205
+ // "core" or "people"), and a public fact sheet uses those words in plain prose ("10 core
206
+ // mechanics", "help other people understand me") (2026-09-21, HQ-1199). So the name is matched only
163
207
  // between slashes, and the id and namespace, which the word match never looked at, are
164
208
  // matched literally. The gap this leaves, stated so the next reader does not rediscover it:
165
209
  // a distinctive circle name written as bare prose passes.
@@ -188,7 +232,7 @@ export function buildPublicExport(evda, { circleName = null, publishedAt = new D
188
232
  }
189
233
  }
190
234
 
191
- return { manifest, files: loaded, sealedCut };
235
+ return { manifest, files: loaded, sealedCut, provenanceCut };
192
236
  }
193
237
 
194
238
  export function canonicalManifest(manifest) {
@@ -259,6 +303,7 @@ export async function publishCommand(dir, circleName, { out, key }) {
259
303
  console.log(`published ${built.files.length} file(s) from the public circle to ${at} (${built.files.length - marked - views.length} by path, ${marked} by mark${views.length ? `, ${views.length} rendered from ${drawn} entries` : ""})`);
260
304
  for (const f of views) console.log(` view ${f.path} rendered from ${f.sources.length} entries (${f.view})`);
261
305
  if (built.sealedCut) console.log(` ${built.sealedCut} sealed section(s) left out of the files they sit in`);
306
+ if (built.provenanceCut) console.log(` ${built.provenanceCut} file(s) exported without their sources: provenance`);
262
307
  console.log(`CHECKED ${built.files.length} public members, signed by ${signed.publicKey}`);
263
308
  return 0;
264
309
  }
package/lib/record.mjs CHANGED
@@ -98,3 +98,57 @@ export function moment(evda, { from, to = from } = {}) {
98
98
  out.entries.sort((a, b) => (a.date === b.date ? a.path.localeCompare(b.path) : a.date.localeCompare(b.date)));
99
99
  return out;
100
100
  }
101
+
102
+ // What a reader needs from each section of a file to know what it was about without opening it
103
+ // (HQ-2324): its heading, its context line (the one line the filing agent wrote above the owner's
104
+ // words, `Context, not their words:` or `*Context (Iris, ...): ...*`), and the opening of the
105
+ // owner's words, cut at a word. A day file's headings are mostly times (`## 20:13`), so a heading
106
+ // alone told a chat app nothing about his last days. Text before the first heading is a section
107
+ // with no heading. Frontmatter, fences and the one-line marker a sealed part leaves are skipped.
108
+ export const OPENING_MAX = 280;
109
+ const CONTEXT_MAX = 240;
110
+ const CONTEXT_LINE = /^\*{0,2}_?\s*Context\b[^:]*:/i;
111
+
112
+ function cutAt(text, max) {
113
+ if (text.length <= max) return text;
114
+ const cut = text.slice(0, max);
115
+ const at = cut.lastIndexOf(" ");
116
+ return `${at > max / 2 ? cut.slice(0, at) : cut}...`;
117
+ }
118
+
119
+ export function sectionDigest(text, { openingMax = OPENING_MAX } = {}) {
120
+ let body = text;
121
+ if (body.startsWith("---")) {
122
+ const end = body.indexOf("\n---", 3);
123
+ if (end !== -1) body = body.slice(end + 4);
124
+ }
125
+ const out = [];
126
+ let cur = { heading: null, lines: [] };
127
+ let fence = false;
128
+ const flush = () => {
129
+ const lines = cur.lines.map((l) => l.trim()).filter((l) => l && !/^\[sealed[:\]]/i.test(l));
130
+ const ctxAt = lines.findIndex((l) => CONTEXT_LINE.test(l));
131
+ const context = ctxAt === -1 ? null : cutAt(lines[ctxAt].replace(/^[*_\s]+|[*_\s]+$/g, ""), CONTEXT_MAX);
132
+ const words = lines.filter((_, i) => i !== ctxAt)
133
+ .map((l) => l.replace(/^>\s?/, "").replace(/^>\s?/, "").trim())
134
+ .filter(Boolean)
135
+ .join(" ")
136
+ .replace(/\s+/g, " ")
137
+ .trim();
138
+ if (cur.heading !== null || context || words) out.push({ heading: cur.heading, context, opening: words ? cutAt(words, openingMax) : null });
139
+ };
140
+ for (const line of body.split("\n")) {
141
+ if (/^\s*(`{3,}|~{3,})/.test(line)) { fence = !fence; continue; }
142
+ if (fence) continue;
143
+ const h = /^#{1,6} (.*)$/.exec(line);
144
+ if (h) { flush(); cur = { heading: h[1].trim(), lines: [] }; continue; }
145
+ cur.lines.push(line);
146
+ }
147
+ flush();
148
+ return out;
149
+ }
150
+
151
+ // The date a heading names (`## 2026-10-02, morning: ...`, `## October 4 (added 2026-10-04)`), or null.
152
+ export function headingDate(heading) {
153
+ return /\b(\d{4}-\d{2}-\d{2})\b/.exec(heading ?? "")?.[1] ?? null;
154
+ }
package/lib/routines.mjs CHANGED
@@ -243,7 +243,17 @@ export function uninstallLaunchd(key, dir, { env = process.env, home = env.HOME
243
243
  // What setup does with the routines it was given (HQ-1246). On a Mac each is installed as a launchd
244
244
  // job; elsewhere nothing can be scheduled from here, so it prints the Claude desktop steps and says
245
245
  // so. Returns the lines to print.
246
- export function installAtSetup(keys, dir, { env = process.env, platform = process.platform } = {}) {
246
+ //
247
+ // `say` prints a line at once, before anything is installed (HQ-1368): a run with --yes asks nothing,
248
+ // so the line that a launchd job is about to be loaded into this Mac is said before launchctl runs,
249
+ // not only in the summary after it. `launchctl` is the tests' stand-in, handed through to
250
+ // installLaunchd with loading on, so a test can watch the order without the real launchctl.
251
+ export function setupNotice(key, dir) {
252
+ const r = ROUTINES[key];
253
+ return `Installing a launchd job on this Mac: ${r.title}, ${r.when.toLowerCase()}, runs ${r.command.join(" ")} on ${resolve(dir)} with your own Claude. Not wanted: create-evda --no-routines, or afterwards evda routine ${key} ${resolve(dir)} --uninstall`;
254
+ }
255
+
256
+ export function installAtSetup(keys, dir, { env = process.env, platform = process.platform, say = null, launchctl = null } = {}) {
247
257
  const lines = [];
248
258
  for (const key of keys) {
249
259
  const r = ROUTINES[key];
@@ -251,7 +261,8 @@ export function installAtSetup(keys, dir, { env = process.env, platform = proces
251
261
  if (platform === "darwin") {
252
262
  let done;
253
263
  try {
254
- done = installLaunchd(key, dir, { env });
264
+ if (say && !falsified(env, "silent-install")) say(setupNotice(key, dir));
265
+ done = installLaunchd(key, dir, launchctl ? { env, launchctl, load: true } : { env });
255
266
  } catch (e) {
256
267
  if (e.code !== "TEMP_RECORD") throw e;
257
268
  lines.push(e.message);
@@ -0,0 +1,169 @@
1
+ // Google Takeout's sidecars (HQ-1368). Takeout writes a `.json` file beside each photo and each Keep
2
+ // note, holding what the app knew about it: when a photo was taken, its caption, when a note was made
3
+ // and last edited. Gathered as files of their own they became captures while their photo or note went
4
+ // somewhere else (the 2026-09-27 trial). So a gather pairs each sidecar with the file it describes,
5
+ // folds the sidecar's date and caption into that file's entry, and never places the sidecar itself.
6
+ //
7
+ // The names Takeout gives them, all paired here:
8
+ // IMG_1.jpg.json the older name
9
+ // IMG_1.jpg.supplemental-metadata.json the newer one
10
+ // IMG_1.jpg.supplemental-metad.json, .supp.json cut short: Takeout caps a name near 51 characters
11
+ // <the first 40-odd characters of a long name>.json the photo's own name cut short
12
+ // IMG_1.jpg(1).json, IMG_1.jpg.supplemental-metadata(1).json the sidecar of IMG_1(1).jpg
13
+ // Note.json a Keep note's, beside Note.html
14
+ // A photo's `-edited` copy has no sidecar of its own and shares its original's.
15
+ //
16
+ // What is folded, and what is not: a photo's date taken (`photoTakenTime.timestamp`, UTC epoch seconds,
17
+ // kept to the second) and whether it has a caption; a Keep note's created and edited times. Its place
18
+ // (`geoData`) and the people Google tagged in it are never read into the proposal or the record. The
19
+ // caption is the owner's words, so the proposal only says there is one; apply reads it again from the
20
+ // sidecar, unchanged since the gather, and keeps it on the record's gathered list beside the photo.
21
+ //
22
+ // pairSidecars(files, read) { folded: Map<rel, facts>, sidecars: [{ rel, why, of }] }
23
+ // captionOf(buf) the caption a photo sidecar holds, or null
24
+ //
25
+ // EVDA_GATHER_FALSIFY=sidecars-as-entries turns the pairing off, so the tests can show their checks fail
26
+ // without it (TAKEOUT_FALSIFY_MODES).
27
+
28
+ import { createHash } from "node:crypto";
29
+ import { basename, dirname, extname } from "node:path";
30
+ import { secretShape } from "./references.mjs";
31
+
32
+ export const TAKEOUT_FALSIFY_MODES = ["sidecars-as-entries"];
33
+ const falsified = (mode) => (process.env.EVDA_GATHER_FALSIFY ?? "").split(",").map((s) => s.trim()).includes(mode);
34
+ const sha256 = (buf) => createHash("sha256").update(buf).digest("hex");
35
+
36
+ // The files a sidecar describes: what Takeout exports from Photos, and Keep's notes.
37
+ const MEDIA_EXT = new Set([".jpg", ".jpeg", ".png", ".gif", ".heic", ".heif", ".webp", ".tif", ".tiff", ".bmp", ".dng", ".raw", ".cr2", ".nef", ".arw", ".mp4", ".mov", ".m4v", ".avi", ".3gp", ".mkv", ".mpg", ".webm", ".mp"]);
38
+ const NOTE_EXT = new Set([".html", ".htm", ".txt", ".md"]);
39
+ const SUPPLEMENTAL = "supplemental-metadata";
40
+ // Takeout's sidecar names run to about 51 characters; a photo's own name is cut from about 46.
41
+ const CUT_AT = 30;
42
+ const SIDECAR_MAX = 1024 * 1024;
43
+
44
+ // The name a sidecar describes, from the sidecar's own name: { target, n, marked }, or null for a name
45
+ // that is not `.json`. `n` is Takeout's "(1)" copy number; `marked` says the name carries the
46
+ // supplemental-metadata marker, whole or cut short.
47
+ export function sidecarTarget(name) {
48
+ if (!/\.json$/i.test(name)) return null;
49
+ let stem = name.slice(0, -5);
50
+ let n = null;
51
+ const copy = /\((\d+)\)$/.exec(stem);
52
+ if (copy) { n = copy[1]; stem = stem.slice(0, copy.index); }
53
+ let marked = false;
54
+ const tail = /\.([a-z-]*)$/i.exec(stem);
55
+ if (tail && SUPPLEMENTAL.startsWith(tail[1].toLowerCase())) {
56
+ const before = stem.slice(0, tail.index);
57
+ if (/\.[a-z0-9]{2,5}$/i.test(before) || tail[1].length >= 4) { stem = before; marked = true; }
58
+ }
59
+ return { target: stem, n, marked };
60
+ }
61
+
62
+ // What kind of Takeout file a parsed JSON value is: "photo", "keep", "album", or null.
63
+ export function takeoutShape(v) {
64
+ if (!v || typeof v !== "object" || Array.isArray(v)) return null;
65
+ if (v.photoTakenTime || (v.creationTime && (v.title !== undefined || v.imageViews !== undefined))) return "photo";
66
+ if (v.createdTimestampUsec !== undefined || v.userEditedTimestampUsec !== undefined) return "keep";
67
+ if (v.albumData || (v.title !== undefined && v.date?.timestamp !== undefined && v.access !== undefined)) return "album";
68
+ return null;
69
+ }
70
+
71
+ // Seconds or microseconds since the epoch, as Takeout writes them, to an ISO time in UTC.
72
+ const iso = (ms) => new Date(ms).toISOString().replace(/\.000Z$/, "Z");
73
+ const seconds = (x) => (/^\d{1,12}$/.test(String(x ?? "").trim()) ? Number(String(x).trim()) : null);
74
+ const usec = (x) => (/^\d{1,18}$/.test(String(x ?? "").trim()) ? Math.floor(Number(String(x).trim()) / 1000) : null);
75
+
76
+ // The facts a sidecar folds into its file's entry. Nothing from geoData or people is read.
77
+ function factsOf(v, shape) {
78
+ const out = {};
79
+ if (shape === "photo") {
80
+ const e = seconds(v.photoTakenTime?.timestamp);
81
+ if (e !== null) { out.takenEpoch = e; out.taken = iso(e * 1000); }
82
+ if (captionText(v)) out.caption = true;
83
+ } else if (shape === "keep") {
84
+ const c = usec(v.createdTimestampUsec);
85
+ const e = usec(v.userEditedTimestampUsec);
86
+ if (c !== null && c > 0) out.created = iso(c);
87
+ if (e !== null && e > 0) out.edited = iso(e);
88
+ }
89
+ return out;
90
+ }
91
+
92
+ // A caption, unless it is empty or shaped like a secret (then it is folded nowhere).
93
+ function captionText(v) {
94
+ const t = typeof v?.description === "string" ? v.description.trim() : "";
95
+ if (!t || t.split("\n").some(secretShape)) return null;
96
+ return t;
97
+ }
98
+
99
+ export function captionOf(buf) {
100
+ try { return captionText(JSON.parse(buf.toString("utf8"))); } catch { return null; }
101
+ }
102
+
103
+ const withN = (name, n) => {
104
+ const ext = extname(name);
105
+ return ext ? `${name.slice(0, -ext.length)}(${n})${ext}` : `${name}(${n})`;
106
+ };
107
+
108
+ // Pair every sidecar in a walked source with the file it describes. `files` are the walk's
109
+ // { rel, abs, size }, `read(abs)` their bytes. Returns { folded, sidecars }: `folded` maps a file's rel
110
+ // to the facts its sidecar gave it, `sidecars` lists every sidecar with why it was not placed and, when
111
+ // it has one, the file it describes (`of`).
112
+ export function pairSidecars(files, read) {
113
+ const folded = new Map();
114
+ const sidecars = [];
115
+ if (falsified("sidecars-as-entries")) return { folded, sidecars };
116
+ const byDir = new Map();
117
+ for (const f of files) {
118
+ const d = dirname(f.rel);
119
+ byDir.set(d, [...(byDir.get(d) ?? []), f]);
120
+ }
121
+ for (const [dir, here] of byDir) {
122
+ const others = here.filter((f) => !/\.json$/i.test(f.rel));
123
+ const names = new Map(others.map((f) => [basename(f.rel), f]));
124
+ for (const f of here) {
125
+ const name = basename(f.rel);
126
+ const t = sidecarTarget(name);
127
+ if (!t) continue;
128
+ let v = null;
129
+ let bad = null;
130
+ if (f.size > SIDECAR_MAX) bad = "too large to be a sidecar";
131
+ else {
132
+ const text = read(f.abs).toString("utf8");
133
+ try { v = JSON.parse(text); } catch { bad = "not valid JSON"; }
134
+ }
135
+ const shape = v ? takeoutShape(v) : null;
136
+ const named = t.marked || MEDIA_EXT.has(extname(t.target).toLowerCase()) || NOTE_EXT.has(extname(t.target).toLowerCase());
137
+ // A .json that is neither shaped nor named like a sidecar is an ordinary file of the pile.
138
+ if (!shape && !named) continue;
139
+ if (shape === "album") { sidecars.push({ rel: f.rel, why: "Google Photos' details of the album this folder holds; not an entry of its own", sidecar: true }); continue; }
140
+ // The file it describes, in this folder.
141
+ let match = null;
142
+ if (t.n) match = names.get(withN(t.target, t.n)) ?? null;
143
+ else match = names.get(t.target) ?? null;
144
+ if (!match && !t.n) {
145
+ const same = others.filter((o) => basename(o.rel, extname(o.rel)) === t.target);
146
+ const notes = same.filter((o) => NOTE_EXT.has(extname(o.rel).toLowerCase()));
147
+ match = (notes.length === 1 ? notes[0] : same.length === 1 ? same[0] : null);
148
+ }
149
+ if (!match && !t.n && t.target.length >= CUT_AT) {
150
+ const cut = others.filter((o) => basename(o.rel).startsWith(t.target));
151
+ if (cut.length === 1) match = cut[0];
152
+ }
153
+ if (!match) {
154
+ sidecars.push({ rel: f.rel, why: bad ? `a Takeout sidecar that is ${bad}, and no file it describes is beside it; nothing read from it` : `a Takeout sidecar for ${t.n ? withN(t.target, t.n) : t.target}, which is not in the pile; placed nowhere`, sidecar: true, unmatched: true });
155
+ continue;
156
+ }
157
+ const facts = bad ? {} : factsOf(v, shape ?? "photo");
158
+ sidecars.push({ rel: f.rel, of: match.rel, why: bad ? `a Takeout sidecar for ${basename(match.rel)} that is ${bad}; nothing read from it, and ${basename(match.rel)} is placed without it` : `Google Takeout's details of ${basename(match.rel)}, folded into its entry`, sidecar: true, ...(bad ? { malformed: true } : {}) });
159
+ if (bad) continue;
160
+ const fold = { sidecar: f.rel, sha256: sha256(read(f.abs)), ...facts };
161
+ folded.set(match.rel, fold);
162
+ // A photo's edited copy has no sidecar of its own: it is the same moment.
163
+ const ext = extname(match.rel);
164
+ const edited = names.get(`${basename(match.rel, ext)}-edited${ext}`);
165
+ if (edited && !folded.has(edited.rel)) folded.set(edited.rel, { ...fold, sharedWith: match.rel });
166
+ }
167
+ }
168
+ return { folded, sidecars };
169
+ }