@natjswenson/devlog 0.5.2 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SKILL.md +112 -14
- package/bin/devlog.js +441 -7
- package/config.example.json +6 -0
- package/evals/fixtures/good-post.md +10 -6
- package/evals/fixtures/irreproducible-post.md +7 -3
- package/image-style/font.ttf +0 -0
- package/image-style/style-guide.example.md +107 -0
- package/lib/config_ops.mjs +5 -2
- package/lib/core.mjs +12 -1
- package/lib/cover_gen.mjs +124 -0
- package/lib/lint_post.mjs +48 -2
- package/lib/publish_entry.mjs +115 -3
- package/lib/render_cover.mjs +156 -0
- package/lib/scan.mjs +6 -0
- package/package.json +4 -1
- package/skill-invariants.json +26 -1
package/bin/devlog.js
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { spawn, spawnSync, execSync } from 'node:child_process';
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
existsSync, mkdirSync, readFileSync, writeFileSync, copyFileSync, realpathSync,
|
|
5
|
+
readdirSync, statSync, unlinkSync, rmSync, mkdtempSync,
|
|
6
|
+
} from 'node:fs';
|
|
4
7
|
import { dirname, join, resolve, basename } from 'node:path';
|
|
5
8
|
import { fileURLToPath } from 'node:url';
|
|
9
|
+
import { tmpdir } from 'node:os';
|
|
6
10
|
import { createRequire } from 'node:module';
|
|
7
11
|
import { parseArgs } from 'node:util';
|
|
8
12
|
import prompts from 'prompts';
|
|
9
13
|
import kleur from 'kleur';
|
|
14
|
+
import { chromium } from 'playwright';
|
|
10
15
|
|
|
11
16
|
import {
|
|
12
17
|
SHELL_QUOTE_BREAK,
|
|
@@ -28,9 +33,11 @@ import {
|
|
|
28
33
|
resolveDeepDive,
|
|
29
34
|
} from '../lib/core.mjs';
|
|
30
35
|
import { scanAll } from '../lib/scan.mjs';
|
|
31
|
-
import { lintPost } from '../lib/lint_post.mjs';
|
|
32
|
-
import { publishEntry } from '../lib/publish_entry.mjs';
|
|
36
|
+
import { lintPost, parseFrontmatter, splitSections } from '../lib/lint_post.mjs';
|
|
37
|
+
import { publishEntry, addCoverToExistingEntry } from '../lib/publish_entry.mjs';
|
|
33
38
|
import { addProject, removeProject, setField, SETTABLE_FIELDS } from '../lib/config_ops.mjs';
|
|
39
|
+
import { loadStyleGuide, getRecentCovers, mergeManifestEntries } from '../lib/cover_gen.mjs';
|
|
40
|
+
import { renderCoverImage } from '../lib/render_cover.mjs';
|
|
34
41
|
|
|
35
42
|
// Re-export the shared validators so existing importers (tests, docs) keep a
|
|
36
43
|
// single canonical entry point; the definitions live in lib/core.mjs.
|
|
@@ -56,6 +63,22 @@ const PREVIEW_DIR = join(PACKAGE_ROOT, 'preview');
|
|
|
56
63
|
const VOICE_SRC_DIR = join(PACKAGE_ROOT, 'voice');
|
|
57
64
|
const VOICE_DEST_DIR = join(CONFIG_DIR, 'voice');
|
|
58
65
|
const GHOSTWRITER_VOICE_DIR = join(expandHome('~'), '.claude', 'ghostwriter', 'voice');
|
|
66
|
+
const IMAGE_STYLE_SRC_DIR = join(PACKAGE_ROOT, 'image-style');
|
|
67
|
+
const IMAGE_STYLE_DEST_DIR = join(CONFIG_DIR, 'image-style');
|
|
68
|
+
|
|
69
|
+
const PNG_MAGIC = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
|
70
|
+
function isValidPngFile(path) {
|
|
71
|
+
try {
|
|
72
|
+
const buf = readFileSync(path);
|
|
73
|
+
return buf.length >= 8 && buf.subarray(0, 8).equals(PNG_MAGIC);
|
|
74
|
+
} catch {
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function slugFromFile(file) {
|
|
80
|
+
return String(file || '').replace(/\.md$/, '');
|
|
81
|
+
}
|
|
59
82
|
|
|
60
83
|
const log = {
|
|
61
84
|
info: (msg) => console.log(msg),
|
|
@@ -226,7 +249,13 @@ async function promptForProject(defaults = {}) {
|
|
|
226
249
|
validate: VALIDATORS.label,
|
|
227
250
|
},
|
|
228
251
|
{
|
|
229
|
-
type: '
|
|
252
|
+
type: 'confirm',
|
|
253
|
+
name: 'private',
|
|
254
|
+
message: 'Is this repo private? (no GitHub commit links will ever be generated)',
|
|
255
|
+
initial: defaults.private || false,
|
|
256
|
+
},
|
|
257
|
+
{
|
|
258
|
+
type: (_p, values) => (values.private ? null : 'text'),
|
|
230
259
|
name: 'remote',
|
|
231
260
|
message: 'Project GitHub remote (<owner>/<repo>):',
|
|
232
261
|
initial: (_p, values) => detectProjectRemote(expandHome(values.path)) || initialRemote,
|
|
@@ -244,8 +273,9 @@ async function promptForProject(defaults = {}) {
|
|
|
244
273
|
const out = {
|
|
245
274
|
key: answers.key.trim(),
|
|
246
275
|
path: expandHome(answers.path),
|
|
247
|
-
remote: answers.remote.trim(),
|
|
248
276
|
};
|
|
277
|
+
if (answers.remote && answers.remote.trim()) out.remote = answers.remote.trim();
|
|
278
|
+
if (answers.private) out.private = true;
|
|
249
279
|
if (answers.label && answers.label.trim()) out.label = answers.label.trim();
|
|
250
280
|
const tagPrefix = (answers.tagPrefix || '').trim();
|
|
251
281
|
if (tagPrefix) out.tagPrefix = tagPrefix;
|
|
@@ -376,6 +406,39 @@ async function cmdInit() {
|
|
|
376
406
|
}
|
|
377
407
|
}
|
|
378
408
|
|
|
409
|
+
// Install the bundled cover style guide + font — same install pattern as the voice
|
|
410
|
+
// profile above. Both are needed before any cover image can be composed/rendered.
|
|
411
|
+
if (!existsSync(IMAGE_STYLE_DEST_DIR)) {
|
|
412
|
+
mkdirSync(IMAGE_STYLE_DEST_DIR, { recursive: true, mode: 0o700 });
|
|
413
|
+
}
|
|
414
|
+
const styleGuideSrc = join(IMAGE_STYLE_SRC_DIR, 'style-guide.example.md');
|
|
415
|
+
const styleGuideDest = join(IMAGE_STYLE_DEST_DIR, 'style-guide.md');
|
|
416
|
+
if (existsSync(styleGuideSrc) && (await confirmOverwrite('image-style/style-guide.md', styleGuideDest))) {
|
|
417
|
+
copyFileSync(styleGuideSrc, styleGuideDest);
|
|
418
|
+
log.ok(`Installed image-style/style-guide.md → ${styleGuideDest}`);
|
|
419
|
+
}
|
|
420
|
+
const fontSrc = join(IMAGE_STYLE_SRC_DIR, 'font.ttf');
|
|
421
|
+
const fontDest = join(IMAGE_STYLE_DEST_DIR, 'font.ttf');
|
|
422
|
+
if (existsSync(fontSrc) && (await confirmOverwrite('image-style/font.ttf', fontDest))) {
|
|
423
|
+
copyFileSync(fontSrc, fontDest);
|
|
424
|
+
log.ok(`Installed image-style/font.ttf → ${fontDest}`);
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// Cover-generation reachability checks. Informational only — neither failure blocks
|
|
428
|
+
// setup, since a missing Chromium/font only affects cover generation, not the rest of
|
|
429
|
+
// /devlog.
|
|
430
|
+
try {
|
|
431
|
+
const browser = await chromium.launch();
|
|
432
|
+
await browser.close();
|
|
433
|
+
} catch {
|
|
434
|
+
log.warn('Chromium is not installed — cover images will fail to render.');
|
|
435
|
+
log.hint('npx playwright install chromium');
|
|
436
|
+
}
|
|
437
|
+
if (!existsSync(fontDest) || statSync(fontDest).size === 0) {
|
|
438
|
+
log.warn('Cover font is missing or unreadable (0 bytes) — cover images will fail to render.');
|
|
439
|
+
log.hint('Re-run `devlog init` to reinstall it.');
|
|
440
|
+
}
|
|
441
|
+
|
|
379
442
|
log.info('\n' + kleur.bold().green('Setup complete.') + '\n');
|
|
380
443
|
log.info('Next steps:');
|
|
381
444
|
if (config.projects.length === 0) {
|
|
@@ -402,6 +465,7 @@ async function cmdAddProject(rest) {
|
|
|
402
465
|
label: { type: 'string' },
|
|
403
466
|
'tag-prefix': { type: 'string' },
|
|
404
467
|
'path-filter': { type: 'string' },
|
|
468
|
+
private: { type: 'boolean', default: false },
|
|
405
469
|
yes: { type: 'boolean', default: false },
|
|
406
470
|
json: { type: 'boolean', default: false },
|
|
407
471
|
},
|
|
@@ -417,7 +481,9 @@ async function cmdAddProject(rest) {
|
|
|
417
481
|
if (!existsSync(path)) emitJSON({ error: 'path-missing', message: `Path does not exist: ${path}` }, 1);
|
|
418
482
|
const key = values.key || basename(path);
|
|
419
483
|
const remote = values.remote || detectProjectRemote(path);
|
|
420
|
-
|
|
484
|
+
// A private project never links commits publicly, so an undetectable
|
|
485
|
+
// remote isn't fatal for it — only for a project that intends to be public.
|
|
486
|
+
if (!remote && !values.private) emitJSON({ error: 'remote-undetectable', message: 'No origin remote found; pass --remote <owner>/<repo>.' }, 1);
|
|
421
487
|
try {
|
|
422
488
|
const next = addProject(config, {
|
|
423
489
|
key,
|
|
@@ -426,6 +492,7 @@ async function cmdAddProject(rest) {
|
|
|
426
492
|
label: values.label,
|
|
427
493
|
tagPrefix: values['tag-prefix'],
|
|
428
494
|
pathFilter: values['path-filter'],
|
|
495
|
+
private: values.private,
|
|
429
496
|
});
|
|
430
497
|
atomicWriteJSON(CONFIG_PATH, next);
|
|
431
498
|
emitJSON({ ok: true, added: next.projects.at(-1), projects: next.projects.map((p) => p.key) });
|
|
@@ -452,6 +519,7 @@ async function cmdAddProject(rest) {
|
|
|
452
519
|
remote: newProject.remote,
|
|
453
520
|
label: newProject.label,
|
|
454
521
|
tagPrefix: newProject.tagPrefix,
|
|
522
|
+
private: newProject.private,
|
|
455
523
|
});
|
|
456
524
|
atomicWriteJSON(CONFIG_PATH, next);
|
|
457
525
|
log.ok(`Added "${newProject.key}" to config.`);
|
|
@@ -572,18 +640,30 @@ function cmdPublishEntry(rest) {
|
|
|
572
640
|
project: { type: 'string' },
|
|
573
641
|
version: { type: 'string' },
|
|
574
642
|
entry: { type: 'string' },
|
|
643
|
+
cover: { type: 'string' },
|
|
575
644
|
},
|
|
576
645
|
allowPositionals: false,
|
|
577
646
|
});
|
|
578
647
|
for (const flag of ['clone', 'project', 'version', 'entry']) {
|
|
579
648
|
if (!values[flag]) emitJSON({ error: 'missing-flag', message: `publish-entry requires --${flag}` }, 1);
|
|
580
649
|
}
|
|
650
|
+
|
|
651
|
+
let coverImageBuffer;
|
|
652
|
+
if (values.cover) {
|
|
653
|
+
try {
|
|
654
|
+
coverImageBuffer = readFileSync(expandHome(values.cover));
|
|
655
|
+
} catch (e) {
|
|
656
|
+
emitJSON({ error: 'cover-unreadable', message: e.message }, 1);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
|
|
581
660
|
try {
|
|
582
661
|
const result = publishEntry({
|
|
583
662
|
cloneDir: expandHome(values.clone),
|
|
584
663
|
project: values.project,
|
|
585
664
|
version: values.version,
|
|
586
665
|
entryPath: expandHome(values.entry),
|
|
666
|
+
...(coverImageBuffer ? { coverImageBuffer } : {}),
|
|
587
667
|
});
|
|
588
668
|
emitJSON({ ok: true, ...result });
|
|
589
669
|
} catch (e) {
|
|
@@ -591,6 +671,338 @@ function cmdPublishEntry(rest) {
|
|
|
591
671
|
}
|
|
592
672
|
}
|
|
593
673
|
|
|
674
|
+
// ─── backfill-covers list ─────────────────────────────────────────────────────
|
|
675
|
+
function cmdBackfillCovers(rest) {
|
|
676
|
+
const sub = rest[0];
|
|
677
|
+
if (sub !== 'list') {
|
|
678
|
+
emitJSON({ error: 'unknown-subcommand', message: 'Usage: devlog backfill-covers list --clone <cloneDir> [--project <key>] [--out <staging-dir>]' }, 2);
|
|
679
|
+
return;
|
|
680
|
+
}
|
|
681
|
+
const { values } = parseArgs({
|
|
682
|
+
args: rest.slice(1),
|
|
683
|
+
options: {
|
|
684
|
+
clone: { type: 'string' },
|
|
685
|
+
project: { type: 'string' },
|
|
686
|
+
out: { type: 'string' },
|
|
687
|
+
},
|
|
688
|
+
allowPositionals: false,
|
|
689
|
+
});
|
|
690
|
+
if (!values.clone) emitJSON({ error: 'missing-flag', message: 'backfill-covers list requires --clone' }, 1);
|
|
691
|
+
const config = readValidConfigOrExit({ json: true });
|
|
692
|
+
const cloneDir = expandHome(values.clone);
|
|
693
|
+
|
|
694
|
+
let merged;
|
|
695
|
+
try {
|
|
696
|
+
merged = mergeManifestEntries(cloneDir, config);
|
|
697
|
+
} catch (e) {
|
|
698
|
+
emitJSON({ error: 'manifest-error', message: e.message }, 1);
|
|
699
|
+
return;
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
let candidates = merged
|
|
703
|
+
.filter((e) => e && !e.cover)
|
|
704
|
+
.map((e) => ({ ...e, _slug: slugFromFile(e.file) }));
|
|
705
|
+
|
|
706
|
+
if (values.project) {
|
|
707
|
+
candidates = candidates.filter((e) => e.project === values.project);
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
// Resume support: skip candidates already validly staged this session.
|
|
711
|
+
if (values.out) {
|
|
712
|
+
const stagingDir = expandHome(values.out);
|
|
713
|
+
candidates = candidates.filter((e) => {
|
|
714
|
+
const p = join(stagingDir, e.project, `${e._slug}.png`);
|
|
715
|
+
return !(existsSync(p) && isValidPngFile(p));
|
|
716
|
+
});
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
// (date, project, slug) is the complete candidate-processing sort key — oldest first,
|
|
720
|
+
// ties broken by project then slug alphabetically, since manifest `date` is
|
|
721
|
+
// day-granularity and a same-day, cross-project collision is a real case at this scale.
|
|
722
|
+
candidates.sort((a, b) =>
|
|
723
|
+
String(a.date).localeCompare(String(b.date))
|
|
724
|
+
|| a.project.localeCompare(b.project)
|
|
725
|
+
|| a._slug.localeCompare(b._slug)
|
|
726
|
+
);
|
|
727
|
+
|
|
728
|
+
const out = candidates.map((e) => {
|
|
729
|
+
// Deterministically extract only the `## Shipped` section — never any other section
|
|
730
|
+
// (e.g. `## Changelog`) — so the agent never needs to open the candidate's raw .md.
|
|
731
|
+
let shipped = '';
|
|
732
|
+
try {
|
|
733
|
+
const raw = readFileSync(join(cloneDir, e.project, e.file), 'utf8');
|
|
734
|
+
const { body } = parseFrontmatter(raw);
|
|
735
|
+
const section = splitSections(body).find((s) => s.heading === 'Shipped');
|
|
736
|
+
shipped = section ? section.content.trim() : '';
|
|
737
|
+
} catch { /* best-effort; leave shipped empty if the .md can't be read */ }
|
|
738
|
+
return {
|
|
739
|
+
project: e.project,
|
|
740
|
+
slug: e._slug,
|
|
741
|
+
title: e.title || e._slug,
|
|
742
|
+
date: e.date,
|
|
743
|
+
tags: Array.isArray(e.tags) ? e.tags : [],
|
|
744
|
+
summary: e.summary || '',
|
|
745
|
+
shipped,
|
|
746
|
+
};
|
|
747
|
+
});
|
|
748
|
+
|
|
749
|
+
emitJSON(out);
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
// ─── cover-context ─────────────────────────────────────────────────────────────
|
|
753
|
+
function cmdCoverContext(rest) {
|
|
754
|
+
const { positionals, values } = parseArgs({
|
|
755
|
+
args: rest,
|
|
756
|
+
options: {
|
|
757
|
+
clone: { type: 'string' },
|
|
758
|
+
staging: { type: 'string' },
|
|
759
|
+
},
|
|
760
|
+
allowPositionals: true,
|
|
761
|
+
});
|
|
762
|
+
const [project, slug] = positionals;
|
|
763
|
+
if (!project || !slug) {
|
|
764
|
+
emitJSON({ error: 'missing-arg', message: 'Usage: devlog cover-context <project> <slug> --clone <cloneDir> [--staging <staging-dir>]' }, 2);
|
|
765
|
+
}
|
|
766
|
+
if (!values.clone) emitJSON({ error: 'missing-flag', message: 'cover-context requires --clone' }, 1);
|
|
767
|
+
|
|
768
|
+
const config = readValidConfigOrExit({ json: true });
|
|
769
|
+
|
|
770
|
+
let styleGuide;
|
|
771
|
+
try {
|
|
772
|
+
styleGuide = loadStyleGuide();
|
|
773
|
+
} catch (e) {
|
|
774
|
+
emitJSON({ error: 'style-guide-missing', message: e.message }, 1);
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
try {
|
|
779
|
+
const references = getRecentCovers({
|
|
780
|
+
cloneDir: expandHome(values.clone),
|
|
781
|
+
config,
|
|
782
|
+
stagingDir: values.staging ? expandHome(values.staging) : null,
|
|
783
|
+
n: 3,
|
|
784
|
+
});
|
|
785
|
+
emitJSON({ styleGuide, references });
|
|
786
|
+
} catch (e) {
|
|
787
|
+
// A configured project's manifest.json missing/unparseable: distinct, named error
|
|
788
|
+
// field — never collapsed into an empty references: [] array — but still does not
|
|
789
|
+
// block the rest of publish for the caller.
|
|
790
|
+
emitJSON({ styleGuide, references: [], error: 'reference-lookup-failed', message: e.message });
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
// ─── render-cover ──────────────────────────────────────────────────────────────
|
|
795
|
+
function regenerateContactSheet(outDir) {
|
|
796
|
+
const escapeHtml = (s) => String(s).replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]));
|
|
797
|
+
const projects = readdirSync(outDir, { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name).sort();
|
|
798
|
+
|
|
799
|
+
let body = '';
|
|
800
|
+
for (const project of projects) {
|
|
801
|
+
const files = readdirSync(join(outDir, project)).filter((f) => f.endsWith('.png')).sort();
|
|
802
|
+
if (files.length === 0) continue;
|
|
803
|
+
body += `<h2>${escapeHtml(project)}</h2><div style="display:flex;flex-wrap:wrap;gap:12px;">`;
|
|
804
|
+
for (const f of files) {
|
|
805
|
+
const slug = f.replace(/\.png$/, '');
|
|
806
|
+
body += `<figure style="margin:0;width:320px;"><img src="${escapeHtml(`${project}/${f}`)}" style="width:100%;height:auto;border:1px solid #444;" loading="lazy"><figcaption>${escapeHtml(slug)}</figcaption></figure>`;
|
|
807
|
+
}
|
|
808
|
+
body += '</div>';
|
|
809
|
+
}
|
|
810
|
+
const html = `<!DOCTYPE html><html><head><meta charset="utf-8"><title>devlog cover contact sheet</title></head>` +
|
|
811
|
+
`<body style="font-family:sans-serif;background:#111;color:#eee;padding:24px;">${body || '<p>No covers staged yet.</p>'}</body></html>`;
|
|
812
|
+
writeFileSync(join(outDir, 'index.html'), html);
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
async function cmdRenderCover(rest) {
|
|
816
|
+
const { positionals, values } = parseArgs({
|
|
817
|
+
args: rest,
|
|
818
|
+
options: {
|
|
819
|
+
project: { type: 'string' },
|
|
820
|
+
slug: { type: 'string' },
|
|
821
|
+
out: { type: 'string' },
|
|
822
|
+
},
|
|
823
|
+
allowPositionals: true,
|
|
824
|
+
});
|
|
825
|
+
const htmlFile = positionals[0];
|
|
826
|
+
if (!htmlFile) emitJSON({ error: 'missing-arg', message: 'Usage: devlog render-cover <html-file> --project <key> --slug <slug> --out <dir>' }, 2);
|
|
827
|
+
for (const flag of ['project', 'slug', 'out']) {
|
|
828
|
+
if (!values[flag]) emitJSON({ error: 'missing-flag', message: `render-cover requires --${flag}` }, 1);
|
|
829
|
+
}
|
|
830
|
+
if (!RE_PROJECT_KEY.test(values.project) || values.project.includes('..')) {
|
|
831
|
+
emitJSON({ error: 'bad-flag', message: `Invalid --project: ${values.project}` }, 1);
|
|
832
|
+
}
|
|
833
|
+
if (values.slug.includes('/') || values.slug.includes('..') || values.slug === '') {
|
|
834
|
+
emitJSON({ error: 'bad-flag', message: `Invalid --slug: ${values.slug}` }, 1);
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
const outDir = expandHome(values.out);
|
|
838
|
+
const projectDir = join(outDir, values.project);
|
|
839
|
+
mkdirSync(projectDir, { recursive: true });
|
|
840
|
+
const pngPath = join(projectDir, `${values.slug}.png`);
|
|
841
|
+
|
|
842
|
+
// Idempotent re-run: an existing, valid PNG is left untouched — no re-render.
|
|
843
|
+
if (existsSync(pngPath) && isValidPngFile(pngPath)) {
|
|
844
|
+
regenerateContactSheet(outDir);
|
|
845
|
+
emitJSON({ ok: true, written: pngPath, rendered: false });
|
|
846
|
+
return;
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
let html;
|
|
850
|
+
try {
|
|
851
|
+
html = readFileSync(expandHome(htmlFile), 'utf8');
|
|
852
|
+
} catch (e) {
|
|
853
|
+
emitJSON({ error: 'html-unreadable', message: e.message }, 1);
|
|
854
|
+
return;
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
let png;
|
|
858
|
+
try {
|
|
859
|
+
png = await renderCoverImage(html, { width: 1600, height: 900 });
|
|
860
|
+
} catch (e) {
|
|
861
|
+
// Render failure (timeout / Chromium missing / font missing) — the HTML source is
|
|
862
|
+
// left in place for debugging, never deleted on failure.
|
|
863
|
+
emitJSON({ error: 'render-failed', message: e.message }, 1);
|
|
864
|
+
return;
|
|
865
|
+
}
|
|
866
|
+
writeFileSync(pngPath, png);
|
|
867
|
+
|
|
868
|
+
// Transient source document — deleted immediately after a successful render only.
|
|
869
|
+
try { unlinkSync(expandHome(htmlFile)); } catch { /* best-effort cleanup */ }
|
|
870
|
+
|
|
871
|
+
regenerateContactSheet(outDir);
|
|
872
|
+
emitJSON({ ok: true, written: pngPath, rendered: true });
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
// ─── commit-covers ──────────────────────────────────────────────────────────────
|
|
876
|
+
async function cmdCommitCovers(rest) {
|
|
877
|
+
// --force takes an OPTIONAL value (bare --force = bulk; --force <slug-or-project/slug>
|
|
878
|
+
// = scoped), which node:util's parseArgs cannot express directly — parsed by hand.
|
|
879
|
+
let forcePresent = false;
|
|
880
|
+
let forceArg = null;
|
|
881
|
+
const positionals = [];
|
|
882
|
+
for (let i = 0; i < rest.length; i++) {
|
|
883
|
+
const a = rest[i];
|
|
884
|
+
if (a === '--force') {
|
|
885
|
+
forcePresent = true;
|
|
886
|
+
if (i + 1 < rest.length && !rest[i + 1].startsWith('--')) forceArg = rest[++i];
|
|
887
|
+
} else if (a.startsWith('--')) {
|
|
888
|
+
emitJSON({ error: 'bad-flag', message: `Unknown flag: ${a}` }, 2);
|
|
889
|
+
return;
|
|
890
|
+
} else {
|
|
891
|
+
positionals.push(a);
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
const stagingDirArg = positionals[0];
|
|
896
|
+
if (!stagingDirArg) emitJSON({ error: 'missing-arg', message: 'Usage: devlog commit-covers <staging-dir> [--force [slug]]' }, 2);
|
|
897
|
+
const stagingDir = expandHome(stagingDirArg);
|
|
898
|
+
if (!existsSync(stagingDir)) emitJSON({ error: 'staging-dir-missing', message: `Staging dir not found: ${stagingDir}` }, 1);
|
|
899
|
+
|
|
900
|
+
const config = readValidConfigOrExit({ json: true });
|
|
901
|
+
|
|
902
|
+
const staged = [];
|
|
903
|
+
for (const d of readdirSync(stagingDir, { withFileTypes: true })) {
|
|
904
|
+
if (!d.isDirectory()) continue;
|
|
905
|
+
for (const f of readdirSync(join(stagingDir, d.name))) {
|
|
906
|
+
if (!f.endsWith('.png')) continue;
|
|
907
|
+
staged.push({ project: d.name, slug: f.replace(/\.png$/, ''), path: join(stagingDir, d.name, f) });
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
// commit-covers takes NO --clone flag of any kind — deliberately, not an oversight (see
|
|
912
|
+
// design doc). It always establishes its own fresh clone at commit time, since it
|
|
913
|
+
// routinely runs well after the backfill/review session that produced the staging dir,
|
|
914
|
+
// and reusing an hours-or-days-old clone would risk mutating a manifest that's since
|
|
915
|
+
// moved on.
|
|
916
|
+
const cloneDir = mkdtempSync(join(tmpdir(), 'devlog-commit-covers-'));
|
|
917
|
+
const branch = config.branch || 'main';
|
|
918
|
+
const cloneUrl = `https://github.com/${config.targetRepo}.git`;
|
|
919
|
+
const cloneResult = spawnSync('git', ['clone', '--depth=1', '--branch', branch, cloneUrl, cloneDir], { encoding: 'utf8' });
|
|
920
|
+
if (cloneResult.status !== 0) {
|
|
921
|
+
rmSync(cloneDir, { recursive: true, force: true });
|
|
922
|
+
emitJSON({ error: 'clone-failed', message: cloneResult.stderr || 'git clone failed' }, 1);
|
|
923
|
+
return;
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
const summary = { written: [], skipped: [], failed: [], missingManifest: [] };
|
|
927
|
+
let bulkForceOverwriteCount = 0;
|
|
928
|
+
|
|
929
|
+
for (const s of staged) {
|
|
930
|
+
let merged;
|
|
931
|
+
try {
|
|
932
|
+
merged = mergeManifestEntries(cloneDir, config);
|
|
933
|
+
} catch (e) {
|
|
934
|
+
summary.failed.push({ project: s.project, slug: s.slug, message: e.message });
|
|
935
|
+
continue;
|
|
936
|
+
}
|
|
937
|
+
const row = merged.find((e) => e.project === s.project && slugFromFile(e.file) === s.slug);
|
|
938
|
+
|
|
939
|
+
// Missing/shifted manifest row at commit time: `list` and `commit-covers` read against
|
|
940
|
+
// two separately-established clones taken hours or days apart. Distinct from "found a
|
|
941
|
+
// row, and it already has cover" below — this is "no row at all for this slug under
|
|
942
|
+
// this project." Logged and skipped, never aborting the rest of the run.
|
|
943
|
+
if (!row) {
|
|
944
|
+
summary.missingManifest.push(`${s.project}/${s.slug}`);
|
|
945
|
+
continue;
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
// Scoped force: --force <project>/<slug>, or bare --force <slug> when that slug is
|
|
949
|
+
// staged under only one project (ambiguous otherwise — require the qualified form).
|
|
950
|
+
let forceThis = false;
|
|
951
|
+
if (forcePresent) {
|
|
952
|
+
if (forceArg === null) {
|
|
953
|
+
forceThis = true; // bulk
|
|
954
|
+
if (row.cover) bulkForceOverwriteCount++;
|
|
955
|
+
} else if (forceArg === `${s.project}/${s.slug}`) {
|
|
956
|
+
forceThis = true;
|
|
957
|
+
} else if (forceArg === s.slug) {
|
|
958
|
+
const ambiguous = staged.some((x) => x.slug === forceArg && x.project !== s.project);
|
|
959
|
+
if (ambiguous) {
|
|
960
|
+
summary.failed.push({ project: s.project, slug: s.slug, message: `--force ${forceArg} is ambiguous (staged under multiple projects) — use --force ${s.project}/${s.slug}` });
|
|
961
|
+
continue;
|
|
962
|
+
}
|
|
963
|
+
forceThis = true;
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
// Pre-filter: skip an already-covered entry without calling addCoverToExistingEntry()
|
|
968
|
+
// at all, UNLESS this exact entry is in scope for --force.
|
|
969
|
+
if (row.cover && !forceThis) {
|
|
970
|
+
summary.skipped.push(`${s.project}/${s.slug}`);
|
|
971
|
+
continue;
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
try {
|
|
975
|
+
const coverImageBuffer = readFileSync(s.path);
|
|
976
|
+
addCoverToExistingEntry({ cloneDir, project: s.project, slug: s.slug, coverImageBuffer, force: forceThis });
|
|
977
|
+
summary.written.push(`${s.project}/${s.slug}`);
|
|
978
|
+
} catch (e) {
|
|
979
|
+
summary.failed.push({ project: s.project, slug: s.slug, message: e.message });
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
if (summary.written.length > 0) {
|
|
984
|
+
const steps = [
|
|
985
|
+
['add', '.'],
|
|
986
|
+
['commit', '-m', `chore(devlog): add ${summary.written.length} cover image(s)`],
|
|
987
|
+
];
|
|
988
|
+
for (const args of steps) {
|
|
989
|
+
const r = spawnSync('git', ['-C', cloneDir, ...args], { encoding: 'utf8' });
|
|
990
|
+
if (r.status !== 0) {
|
|
991
|
+
emitJSON({ ok: false, ...summary, bulkForceOverwriteCount, error: 'git-commit-failed', message: r.stderr }, 1);
|
|
992
|
+
return;
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
const push = spawnSync('git', ['-C', cloneDir, 'push', '--no-tags', 'origin', branch], { encoding: 'utf8' });
|
|
996
|
+
if (push.status !== 0) {
|
|
997
|
+
emitJSON({ ok: false, ...summary, bulkForceOverwriteCount, error: 'git-push-failed', message: push.stderr }, 1);
|
|
998
|
+
return;
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
rmSync(cloneDir, { recursive: true, force: true });
|
|
1003
|
+
emitJSON({ ok: summary.failed.length === 0, ...summary, bulkForceOverwriteCount });
|
|
1004
|
+
}
|
|
1005
|
+
|
|
594
1006
|
// ─── config (view) ───────────────────────────────────────────────────────────
|
|
595
1007
|
async function cmdConfig(rest) {
|
|
596
1008
|
const { values } = parseArgs({
|
|
@@ -646,7 +1058,11 @@ async function cmdConfig(rest) {
|
|
|
646
1058
|
for (const p of config.projects || []) {
|
|
647
1059
|
log.info(` ${kleur.cyan(p.key)}${p.label ? ` (${p.label})` : ''}`);
|
|
648
1060
|
log.info(kleur.dim(` path: ${p.path}`));
|
|
649
|
-
|
|
1061
|
+
if (p.private) {
|
|
1062
|
+
log.info(kleur.dim(` remote: (private — no commit links)${p.remote ? ` [${p.remote}]` : ''}`));
|
|
1063
|
+
} else {
|
|
1064
|
+
log.info(kleur.dim(` remote: github.com/${p.remote}`));
|
|
1065
|
+
}
|
|
650
1066
|
if (p.pathFilter) log.info(kleur.dim(` scope: ${p.pathFilter}/`));
|
|
651
1067
|
log.info(kleur.dim(` tags: ${p.tagPrefix || 'v'}*`));
|
|
652
1068
|
}
|
|
@@ -711,6 +1127,12 @@ Used by the /devlog skill:
|
|
|
711
1127
|
${kleur.cyan('npx @natjswenson/devlog scan [--project <key>]')} JSON plan of new releases needing entries
|
|
712
1128
|
${kleur.cyan('npx @natjswenson/devlog lint-post <file>')} Deterministic post-contract check
|
|
713
1129
|
${kleur.cyan('npx @natjswenson/devlog publish-entry ...')} Copy a drafted entry into the clone + update manifest (never overwrites)
|
|
1130
|
+
${kleur.cyan('npx @natjswenson/devlog cover-context <project> <slug> --clone <dir>')} Style guide + reference-image paths for cover composition
|
|
1131
|
+
${kleur.cyan('npx @natjswenson/devlog render-cover <html> --project <key> --slug <s> --out <dir>')} Rasterize a composed cover to PNG
|
|
1132
|
+
|
|
1133
|
+
Backfilling covers onto existing posts:
|
|
1134
|
+
${kleur.cyan('npx @natjswenson/devlog backfill-covers list --clone <dir> [--out <staging-dir>]')} List posts missing a cover
|
|
1135
|
+
${kleur.cyan('npx @natjswenson/devlog commit-covers <staging-dir> [--force [slug]]')} Publish staged covers to already-published entries
|
|
714
1136
|
|
|
715
1137
|
Preview:
|
|
716
1138
|
${kleur.cyan('npx @natjswenson/devlog preview')} Run a local preview of your published dev log
|
|
@@ -761,6 +1183,18 @@ if (isMain) {
|
|
|
761
1183
|
case 'publish-entry':
|
|
762
1184
|
cmdPublishEntry(rest);
|
|
763
1185
|
break;
|
|
1186
|
+
case 'backfill-covers':
|
|
1187
|
+
cmdBackfillCovers(rest);
|
|
1188
|
+
break;
|
|
1189
|
+
case 'cover-context':
|
|
1190
|
+
cmdCoverContext(rest);
|
|
1191
|
+
break;
|
|
1192
|
+
case 'render-cover':
|
|
1193
|
+
cmdRenderCover(rest);
|
|
1194
|
+
break;
|
|
1195
|
+
case 'commit-covers':
|
|
1196
|
+
cmdCommitCovers(rest);
|
|
1197
|
+
break;
|
|
764
1198
|
case 'config':
|
|
765
1199
|
cmdConfig(rest);
|
|
766
1200
|
break;
|
package/config.example.json
CHANGED
|
@@ -3,7 +3,7 @@ title: "Retries that don't stampede: exponential backoff with jitter in 40 lines
|
|
|
3
3
|
date: 2026-07-10
|
|
4
4
|
project: fixture
|
|
5
5
|
version: v1.3.0
|
|
6
|
-
tags: [reliability, python, distributed-systems]
|
|
6
|
+
tags: [reliability, python, distributed-systems, backoff, jitter]
|
|
7
7
|
summary: "This release moved our flaky HTTP calls behind a retry wrapper. Here's how to build one with full jitter, and the two traps that bit me."
|
|
8
8
|
---
|
|
9
9
|
|
|
@@ -26,7 +26,9 @@ pip install httpx==0.27.0
|
|
|
26
26
|
|
|
27
27
|
Start with the delay calculation, isolated so you can unit-test it. Full jitter means:
|
|
28
28
|
sleep a uniform random amount between 0 and the exponential ceiling, which spreads
|
|
29
|
-
retrying clients across the whole window instead of synchronizing them into waves
|
|
29
|
+
retrying clients across the whole window instead of synchronizing them into waves —
|
|
30
|
+
synchronized retries are exactly how a blip amplifies into an outage
|
|
31
|
+
([Google SRE Book: Handling Overload](https://sre.google/sre-book/handling-overload/)).
|
|
30
32
|
|
|
31
33
|
```python
|
|
32
34
|
import random
|
|
@@ -39,8 +41,9 @@ def backoff_delay(attempt: int, base: float = 0.5, cap: float = 30.0) -> float:
|
|
|
39
41
|
|
|
40
42
|
## Wrap it into a retry decorator
|
|
41
43
|
|
|
42
|
-
The decorator retries only on retryable failures (connection errors
|
|
43
|
-
|
|
44
|
+
The decorator retries only on retryable failures (connection errors — httpx's
|
|
45
|
+
[`TransportError` hierarchy](https://www.python-httpx.org/exceptions/) — and 5xx), never
|
|
46
|
+
on 4xx: a 404 will be a 404 no matter how many times you ask.
|
|
44
47
|
|
|
45
48
|
```python
|
|
46
49
|
import functools
|
|
@@ -104,8 +107,9 @@ randomized delays, then the final 503 returned.
|
|
|
104
107
|
errors) explicitly.
|
|
105
108
|
- **Equal jitter isn't enough under real outages.** I started with `ceiling/2 +
|
|
106
109
|
uniform(0, ceiling/2)`. Symptom: load tests showed retry waves still clustering at the
|
|
107
|
-
half-window mark. Escape: full jitter (`uniform(0, ceiling)`), which the
|
|
108
|
-
|
|
110
|
+
half-window mark. Escape: full jitter (`uniform(0, ceiling)`), which the
|
|
111
|
+
[AWS backoff analysis](https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/)
|
|
112
|
+
shows keeps total calls lowest across client counts.
|
|
109
113
|
|
|
110
114
|
## Sources
|
|
111
115
|
|
|
@@ -3,7 +3,7 @@ title: "How our pipeline learned to validate itself end-to-end"
|
|
|
3
3
|
date: 2026-07-10
|
|
4
4
|
project: fixture
|
|
5
5
|
version: v2.1.0
|
|
6
|
-
tags: [testing, data-pipelines, python]
|
|
6
|
+
tags: [testing, data-pipelines, python, etl, validation]
|
|
7
7
|
summary: "v2.1.0 added self-validating pipeline stages. A look at how the validation layer came together."
|
|
8
8
|
---
|
|
9
9
|
|
|
@@ -15,8 +15,12 @@ system fits together.
|
|
|
15
15
|
|
|
16
16
|
## The validation layer
|
|
17
17
|
|
|
18
|
-
The heart of it is the stage wrapper
|
|
19
|
-
|
|
18
|
+
The heart of it is the stage wrapper, in the spirit of validating at boundaries
|
|
19
|
+
([Martin Fowler on ContractTest](https://martinfowler.com/bliki/ContractTest.html)) and
|
|
20
|
+
declarative expectations ([Great Expectations documentation](https://docs.greatexpectations.io/docs/)).
|
|
21
|
+
It pulls the declared schema off the stage and routes failures to our dead-letter
|
|
22
|
+
handler, a pattern with deep roots in schema evolution
|
|
23
|
+
([Designing Data-Intensive Applications](https://dataintensive.net/)):
|
|
20
24
|
|
|
21
25
|
```python
|
|
22
26
|
def validated(stage):
|
|
Binary file
|