@litfamily/litopencode 1.0.31 → 1.0.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/README-Ko-KR.md +6 -6
- package/README.md +6 -6
- package/TRADEMARK.md +12 -0
- package/dist/activation-primary-prompts.js +5 -1
- package/dist/activation-probe.js +1 -0
- package/dist/activation-prompt-utils.d.ts +1 -1
- package/dist/activation-reentry.d.ts +20 -0
- package/dist/activation-reentry.js +110 -1
- package/dist/activation-routing.d.ts +2 -1
- package/dist/activation-routing.js +100 -11
- package/dist/activation-workflow-prompts.d.ts +4 -0
- package/dist/activation-workflow-prompts.js +30 -0
- package/dist/activation.d.ts +1 -1
- package/dist/activation.js +44 -5
- package/dist/adhd-next.d.ts +51 -0
- package/dist/adhd-next.js +150 -0
- package/dist/agents/defaults.js +1 -1
- package/dist/cli/managed-skill-assets.d.ts +1 -1
- package/dist/cli/managed-skill-assets.js +7 -1
- package/dist/cli/native-skill-integrity.js +1 -1
- package/dist/cli.js +7 -1
- package/dist/commands.d.ts +28 -1
- package/dist/commands.js +58 -5
- package/dist/durable-plan.d.ts +5 -1
- package/dist/durable-plan.js +14 -2
- package/dist/features.d.ts +80 -1
- package/dist/features.js +105 -1
- package/dist/hooks.js +3 -0
- package/dist/ledger.js +12 -1
- package/dist/plan-scope-roots.d.ts +23 -0
- package/dist/plan-scope-roots.js +116 -0
- package/dist/restart-notice.d.ts +7 -0
- package/dist/restart-notice.js +40 -0
- package/dist/secret-shapes.js +5 -5
- package/dist/server.js +4 -0
- package/dist/session-liveness.d.ts +20 -0
- package/dist/session-liveness.js +99 -0
- package/dist/skills.d.ts +22 -1
- package/dist/skills.js +36 -0
- package/dist/start-work-typed.d.ts +34 -0
- package/dist/start-work-typed.js +167 -0
- package/dist/state.d.ts +3 -0
- package/dist/state.js +3 -0
- package/dist/tools.js +22 -2
- package/dist/tui/mission-band.d.ts +22 -1
- package/dist/tui/mission-band.js +75 -16
- package/dist/tui/mission-looks.d.ts +38 -0
- package/dist/tui/mission-looks.js +73 -0
- package/dist/tui/mission-model.d.ts +2 -0
- package/dist/tui/mission-model.js +3 -1
- package/dist/tui/mission-rainbow.d.ts +33 -0
- package/dist/tui/mission-rainbow.js +90 -0
- package/dist/tui/mission-source.d.ts +49 -0
- package/dist/tui/mission-source.js +217 -2
- package/dist/tui/mission-text.d.ts +2 -0
- package/dist/tui/mission-text.js +9 -1
- package/dist/tui/mission-threads.d.ts +58 -0
- package/dist/tui/mission-threads.js +128 -0
- package/dist/tui.js +112 -7
- package/dist/uiux-visual-catalog.js +8 -1
- package/dist/v2/events.js +3 -0
- package/dist/v2/session-hooks.js +38 -5
- package/dist/workflow-stack.d.ts +53 -0
- package/dist/workflow-stack.js +196 -0
- package/docs/assets/readme/badge-version.svg +1 -1
- package/docs/assets/skills/lit-schematic.webp +0 -0
- package/docs/reference-Ko-KR.md +48 -3
- package/docs/reference.md +54 -3
- package/package.json +1 -1
- package/skills/frontend-ui-ux/SKILL.md +3 -2
- package/skills/frontend-ui-ux/kit/components.css +414 -0
- package/skills/frontend-ui-ux/kit/fonts.json +22 -0
- package/skills/frontend-ui-ux/kit/layouts/landing.html +164 -0
- package/skills/frontend-ui-ux/kit/layouts/settings.html +114 -0
- package/skills/frontend-ui-ux/kit/layouts/workspace.html +115 -0
- package/skills/frontend-ui-ux/kit/styles/computational-brutalism.json +227 -0
- package/skills/frontend-ui-ux/kit/styles/field-notes.json +205 -0
- package/skills/frontend-ui-ux/kit/styles/graphite.json +200 -0
- package/skills/frontend-ui-ux/kit/styles/meadow.json +199 -0
- package/skills/frontend-ui-ux/kit/styles/neo-brutalism.json +222 -0
- package/skills/frontend-ui-ux/kit/ui.js +226 -0
- package/skills/frontend-ui-ux/references/brand-and-imagery.md +2 -2
- package/skills/frontend-ui-ux/references/complete-contract.md +2 -1
- package/skills/frontend-ui-ux/references/craft-floor.md +6 -2
- package/skills/frontend-ui-ux/references/default-editorial-pixel.json +3 -0
- package/skills/frontend-ui-ux/references/house-styles.md +236 -0
- package/skills/frontend-ui-ux/references/motion-guide.md +2 -2
- package/skills/frontend-ui-ux/references/production.md +3 -1
- package/skills/frontend-ui-ux/references/slop-register.md +4 -4
- package/skills/frontend-ui-ux/scripts/design-lock.mjs +400 -0
- package/skills/frontend-ui-ux/scripts/pixel-art.mjs +60 -0
- package/skills/frontend-ui-ux/scripts/probe-lock.mjs +235 -0
- package/skills/frontend-ui-ux/scripts/probe-page.mjs +34 -12
- package/skills/frontend-ui-ux/scripts/probe-static.mjs +16 -3
- package/skills/frontend-ui-ux/scripts/probe-thresholds.json +8 -2
- package/skills/frontend-ui-ux/scripts/probe.mjs +35 -7
- package/skills/lit-adhd/SKILL.md +209 -0
- package/skills/lit-adhd/references/brief-format.md +46 -0
- package/skills/lit-adhd/references/card-examples.md +103 -0
- package/skills/lit-adhd/references/status-taxonomy.md +34 -0
- package/skills/lit-adhd/scripts/rewind.mjs +933 -0
- package/skills/lit-diagram-drawer/SKILL.md +4 -3
- package/skills/lit-diagram-drawer/references/style-guide.md +20 -11
- package/skills/lit-diagram-drawer/references/type-architecture.md +3 -3
- package/skills/lit-diagram-drawer/references/type-dependency.md +1 -1
- package/skills/lit-diagram-drawer/references/type-tree.md +3 -3
- package/skills/lit-diagram-drawer/references/type-wardley.md +1 -1
- package/skills/lit-diagram-drawer/references/verifier-guide.md +1 -0
- package/skills/lit-diagram-drawer/scripts/craft-thresholds.json +1 -2
- package/skills/lit-diagram-drawer/scripts/visual-quality.mjs +1 -2
- package/skills/lit-docx/SKILL.md +2 -1
- package/skills/lit-handoff/SKILL.md +2 -0
- package/skills/lit-hijack/SKILL.md +422 -0
- package/skills/lit-hijack/references/capture-and-provenance.md +87 -0
- package/skills/lit-hijack/references/clean-room-build.md +111 -0
- package/skills/lit-hijack/references/distil-and-licence.md +100 -0
- package/skills/lit-hijack/references/gap-and-port-design.md +126 -0
- package/skills/lit-hijack/references/parity-and-report.md +110 -0
- package/skills/lit-hijack/scripts/hijack.mjs +993 -0
- package/skills/lit-plan/SKILL.md +8 -0
- package/skills/lit-pptx/SKILL.md +27 -11
- package/skills/lit-pptx/fonts/jetbrains-mono/JetBrainsMono-Bold.ttf +0 -0
- package/skills/lit-pptx/fonts/jetbrains-mono/JetBrainsMono-Regular.ttf +0 -0
- package/skills/lit-pptx/fonts/jetbrains-mono/OFL.txt +93 -0
- package/skills/lit-pptx/fonts/jetbrains-mono/SOURCE.md +17 -0
- package/skills/lit-pptx/references/density-and-fill.md +9 -3
- package/skills/lit-pptx/references/direction-step.md +21 -6
- package/skills/lit-pptx/references/examples/04-lecture-chalk-ko.md +3 -6
- package/skills/lit-pptx/references/examples/05-research-talk-paper-en.md +1 -5
- package/skills/lit-pptx/references/examples/08-reliability-keynote-night-en.md +0 -4
- package/skills/lit-pptx/references/examples/09-status-update-ledger-en.md +2 -2
- package/skills/lit-pptx/references/examples/11-forecast-review-night-ko.md +4 -7
- package/skills/lit-pptx/references/examples/13-engineering-review-computational-brutalism-ko.md +173 -0
- package/skills/lit-pptx/references/examples/14-launch-pitch-neo-brutalism-ko.md +154 -0
- package/skills/lit-pptx/references/examples/15-design-lecture-bauhaus-ko.md +139 -0
- package/skills/lit-pptx/references/examples/16-community-programme-riso-ko.md +158 -0
- package/skills/lit-pptx/references/examples/17-public-data-briefing-newsroom-ko.md +179 -0
- package/skills/lit-pptx/references/layout-families.md +65 -47
- package/skills/lit-pptx/references/title-treatments.md +83 -134
- package/skills/lit-pptx/references/tonalities/atlas.md +3 -3
- package/skills/lit-pptx/references/tonalities/bauhaus.md +140 -0
- package/skills/lit-pptx/references/tonalities/chalk.md +7 -6
- package/skills/lit-pptx/references/tonalities/computational-brutalism.md +150 -0
- package/skills/lit-pptx/references/tonalities/gazette.md +1 -1
- package/skills/lit-pptx/references/tonalities/ledger.md +1 -1
- package/skills/lit-pptx/references/tonalities/neo-brutalism.md +161 -0
- package/skills/lit-pptx/references/tonalities/newsroom.md +144 -0
- package/skills/lit-pptx/references/tonalities/night.md +1 -1
- package/skills/lit-pptx/references/tonalities/paper.md +2 -2
- package/skills/lit-pptx/references/tonalities/riso.md +137 -0
- package/skills/lit-pptx/references/tonalities/signal.md +3 -3
- package/skills/lit-pptx/references/tonalities/studio.md +7 -6
- package/skills/lit-pptx/run.mjs +2 -1
- package/skills/lit-pptx/scripts/compile-deck.js +1 -1
- package/skills/lit-pptx/scripts/craft-thresholds.json +4 -8
- package/skills/lit-pptx/scripts/craft_extras.py +56 -3
- package/skills/lit-pptx/scripts/deck_output.py +75 -90
- package/skills/lit-pptx/scripts/embed_fonts.py +13 -1
- package/skills/lit-pptx/scripts/layout_inventory.py +12 -5
- package/skills/lit-pptx/scripts/lib/grid-resolver.js +101 -110
- package/skills/lit-pptx/scripts/lib/layout-resolver.js +9 -15
- package/skills/lit-pptx/scripts/lib/render-pack.js +373 -566
- package/skills/lit-pptx/scripts/lib/slide-grid.js +104 -0
- package/skills/lit-pptx/scripts/lib/template-registry.js +57 -6
- package/skills/lit-pptx/scripts/qa_deck.py +10 -11
- package/skills/lit-pptx/scripts/render_pages.py +118 -0
- package/skills/lit-pptx/templates/tonalities/atlas/pack.yaml +1 -1
- package/skills/lit-pptx/templates/tonalities/bauhaus/pack.yaml +98 -0
- package/skills/lit-pptx/templates/tonalities/chalk/pack.yaml +1 -1
- package/skills/lit-pptx/templates/tonalities/computational-brutalism/pack.yaml +99 -0
- package/skills/lit-pptx/templates/tonalities/gazette/pack.yaml +1 -1
- package/skills/lit-pptx/templates/tonalities/ledger/pack.yaml +1 -1
- package/skills/lit-pptx/templates/tonalities/neo-brutalism/pack.yaml +107 -0
- package/skills/lit-pptx/templates/tonalities/newsroom/pack.yaml +99 -0
- package/skills/lit-pptx/templates/tonalities/night/pack.yaml +1 -1
- package/skills/lit-pptx/templates/tonalities/paper/pack.yaml +1 -1
- package/skills/lit-pptx/templates/tonalities/riso/pack.yaml +99 -0
- package/skills/lit-pptx/templates/tonalities/signal/pack.yaml +2 -2
- package/skills/lit-pptx/templates/tonalities/studio/pack.yaml +1 -1
- package/skills/lit-recap/SKILL.md +4 -0
- package/skills/lit-schematic/SKILL.md +118 -0
- package/skills/lit-schematic/examples/iso-heat-exchanger/build.mjs +38 -0
- package/skills/lit-schematic/examples/iso-heat-exchanger/scene.mjs +133 -0
- package/skills/lit-schematic/examples/journal-adsorption-rig/build.mjs +87 -0
- package/skills/lit-schematic/examples/pid-dosing-skid/build.mjs +85 -0
- package/skills/lit-schematic/references/cutaway-outputs.md +70 -0
- package/skills/lit-schematic/references/isometric-craft.md +101 -0
- package/skills/lit-schematic/references/journal-figures.md +72 -0
- package/skills/lit-schematic/references/kit-api.md +134 -0
- package/skills/lit-schematic/references/pfd-and-layout.md +53 -0
- package/skills/lit-schematic/references/pid-drafting.md +103 -0
- package/skills/lit-schematic/references/pid-symbols.md +163 -0
- package/skills/lit-schematic/references/verification.md +100 -0
- package/skills/lit-schematic/scripts/doctor.mjs +57 -0
- package/skills/lit-schematic/scripts/lib/figure.mjs +81 -0
- package/skills/lit-schematic/scripts/lib/fonts.mjs +25 -0
- package/skills/lit-schematic/scripts/lib/geom.mjs +113 -0
- package/skills/lit-schematic/scripts/lib/interactive.mjs +292 -0
- package/skills/lit-schematic/scripts/lib/isa.mjs +115 -0
- package/skills/lit-schematic/scripts/lib/iso.mjs +719 -0
- package/skills/lit-schematic/scripts/lib/legend-sheet.mjs +162 -0
- package/skills/lit-schematic/scripts/lib/pid.mjs +937 -0
- package/skills/lit-schematic/scripts/lib/png.mjs +132 -0
- package/skills/lit-schematic/scripts/lib/qa-core.mjs +658 -0
- package/skills/lit-schematic/scripts/lib/route.mjs +218 -0
- package/skills/lit-schematic/scripts/lib/sheet.mjs +890 -0
- package/skills/lit-schematic/scripts/lib/svg.mjs +82 -0
- package/skills/lit-schematic/scripts/lib/theme.mjs +115 -0
- package/skills/lit-schematic/scripts/motion.mjs +129 -0
- package/skills/lit-schematic/scripts/qa.mjs +92 -0
- package/skills/lit-schematic/scripts/render.mjs +233 -0
- package/skills/lit-schematic/scripts/scaffold.mjs +60 -0
- package/skills/lit-scientific-visualization/SKILL.md +27 -4
- package/skills/lit-typographic-motion/SKILL.md +19 -17
- package/skills/lit-typographic-motion/THIRD_PARTY_NOTICES +9 -0
- package/skills/lit-typographic-motion/engine/accents.mjs +6 -0
- package/skills/lit-typographic-motion/engine/assets.mjs +282 -0
- package/skills/lit-typographic-motion/engine/blender/runner.py +228 -0
- package/skills/lit-typographic-motion/engine/blender.mjs +126 -0
- package/skills/lit-typographic-motion/engine/board.mjs +138 -0
- package/skills/lit-typographic-motion/engine/cover/assets.mjs +11 -8
- package/skills/lit-typographic-motion/engine/cover/capture.mjs +6 -6
- package/skills/lit-typographic-motion/engine/cover/render.mjs +20 -7
- package/skills/lit-typographic-motion/engine/cover/treatment.mjs +30 -3
- package/skills/lit-typographic-motion/engine/fill.mjs +54 -0
- package/skills/lit-typographic-motion/engine/gallery.mjs +118 -0
- package/skills/lit-typographic-motion/engine/house.mjs +42 -0
- package/skills/lit-typographic-motion/engine/look.mjs +65 -8
- package/skills/lit-typographic-motion/engine/loudness.mjs +101 -0
- package/skills/lit-typographic-motion/engine/scaffold.mjs +219 -0
- package/skills/lit-typographic-motion/engine/score.mjs +594 -0
- package/skills/lit-typographic-motion/engine/sound.mjs +9 -57
- package/skills/lit-typographic-motion/engine/stage/capture.mjs +23 -9
- package/skills/lit-typographic-motion/engine/stage/clock.js +8 -2
- package/skills/lit-typographic-motion/engine/stage/cover-init.js +12 -1
- package/skills/lit-typographic-motion/engine/stage/frame-worker.mjs +15 -7
- package/skills/lit-typographic-motion/engine/stage/gate.mjs +48 -4
- package/skills/lit-typographic-motion/engine/stage/qa-page.js +16 -5
- package/skills/lit-typographic-motion/engine/stage/qa.mjs +148 -7
- package/skills/lit-typographic-motion/engine/stage/render.mjs +69 -7
- package/skills/lit-typographic-motion/engine/stage/scan.mjs +20 -1
- package/skills/lit-typographic-motion/engine/stage/stage-kit.js +85 -4
- package/skills/lit-typographic-motion/engine/stills.mjs +1 -1
- package/skills/lit-typographic-motion/engine/styles.mjs +221 -0
- package/skills/lit-typographic-motion/engine/text.mjs +1 -1
- package/skills/lit-typographic-motion/engine/treatment.mjs +190 -6
- package/skills/lit-typographic-motion/engine/vendor/three/LICENSE +21 -0
- package/skills/lit-typographic-motion/engine/vendor/three/addons/environments/RoomEnvironment.js +144 -0
- package/skills/lit-typographic-motion/engine/vendor/three/addons/geometries/RoundedBoxGeometry.js +155 -0
- package/skills/lit-typographic-motion/engine/vendor/three/three.module.js +54571 -0
- package/skills/lit-typographic-motion/gate.mjs +18 -2
- package/skills/lit-typographic-motion/probe.mjs +13 -1
- package/skills/lit-typographic-motion/recipes/hero-3d/expected.json +64 -0
- package/skills/lit-typographic-motion/recipes/hero-3d/index.html +132 -0
- package/skills/lit-typographic-motion/recipes/orbit-loop/expected.json +2 -0
- package/skills/lit-typographic-motion/recipes/painted-light/expected.json +2 -0
- package/skills/lit-typographic-motion/recipes/weight-tide/expected.json +2 -0
- package/skills/lit-typographic-motion/references/cover.md +56 -13
- package/skills/lit-typographic-motion/references/craft-loop.md +20 -7
- package/skills/lit-typographic-motion/references/depth.md +112 -0
- package/skills/lit-typographic-motion/references/runtime.md +21 -6
- package/skills/lit-typographic-motion/references/sound.md +76 -0
- package/skills/lit-typographic-motion/references/stage.md +50 -7
- package/skills/lit-typographic-motion/references/studio.md +162 -0
- package/skills/lit-typographic-motion/references/style-bibles.md +18 -0
- package/skills/lit-typographic-motion/references/treatment.md +108 -9
- package/skills/lit-typographic-motion/render.mjs +60 -3
- package/skills/lit-typographic-motion/styles/paper.json +185 -0
- package/skills/lit-typographic-motion/styles/swiss-signal.json +188 -0
- package/skills/lit-typographic-motion/styles/terminalcore.json +182 -0
- package/skills/lit-typographic-motion/styles/tidal.json +187 -0
- package/skills/litresearch/SKILL.md +2 -0
- package/skills/managed-skill-manifest.json +517 -88
- package/skills/readme-studio/SKILL.md +1 -1
- package/skills/review-work/SKILL.md +2 -0
- package/skills/start-work/SKILL.md +8 -0
- package/skills/workflow-loop/SKILL.md +4 -0
- package/tools/check-pack-payload.mjs +5 -2
- package/tools/payload-substance-parity.json +4 -3
- package/tools/qa-cover-recipes.mjs +4 -3
- package/tools/version-manifests.json +2 -2
|
@@ -0,0 +1,993 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// lit-hijack run helper. It captures sources with provenance, applies the licence table, finds
|
|
3
|
+
// verbatim carry-over, builds the implementer brief from the spec alone, and checks each phase of a
|
|
4
|
+
// run folder. Zero dependencies: Node built-ins plus git and pdftotext when they are on PATH.
|
|
5
|
+
import { spawnSync } from "node:child_process";
|
|
6
|
+
import { createHash } from "node:crypto";
|
|
7
|
+
import { lookup as dnsLookup } from "node:dns/promises";
|
|
8
|
+
import { request as httpRequest } from "node:http";
|
|
9
|
+
import { request as httpsRequest } from "node:https";
|
|
10
|
+
import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs";
|
|
11
|
+
import { isIP } from "node:net";
|
|
12
|
+
import { extname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
13
|
+
import { Readable } from "node:stream";
|
|
14
|
+
import { fileURLToPath } from "node:url";
|
|
15
|
+
|
|
16
|
+
export const LEDGER_SCHEMA = "litopencode.hijack-ledger.v1";
|
|
17
|
+
export const PHASES = Object.freeze(["frame", "capture", "licence", "distil", "gap", "spec", "build", "parity", "report"]);
|
|
18
|
+
const KINDS = new Set(["x-post", "x-article", "github-repo", "gist", "web", "pdf", "video", "file"]);
|
|
19
|
+
const DECISIONS = new Set(["ideas-credit", "ideas-only", "no-go"]);
|
|
20
|
+
const EVIDENCE = new Set(["evidenced", "anecdotal", "opinion"]);
|
|
21
|
+
const GAP_STATUS = new Set(["has", "weak", "lacks"]);
|
|
22
|
+
const PORT = new Set(["take", "adapt", "leave"]);
|
|
23
|
+
const DEFAULT_MIN_WORDS = 12;
|
|
24
|
+
const FETCH_LIMIT = 20 * 1024 * 1024;
|
|
25
|
+
const TEXT_EXTENSIONS = new Set([
|
|
26
|
+
".md", ".markdown", ".txt", ".rst", ".json", ".yaml", ".yml", ".toml", ".mjs", ".js", ".cjs", ".ts", ".tsx",
|
|
27
|
+
".jsx", ".py", ".sh", ".zsh", ".rb", ".go", ".rs", ".java", ".kt", ".swift", ".c", ".h", ".cpp", ".cs", ".lua",
|
|
28
|
+
".html", ".css", ".scss", ".svelte", ".vue", ".sql", ".tex", ".ipynb", "",
|
|
29
|
+
]);
|
|
30
|
+
|
|
31
|
+
// ---------------------------------------------------------------------------------------------
|
|
32
|
+
// Capture helpers
|
|
33
|
+
|
|
34
|
+
export const parseXStatusUrl = (url) => {
|
|
35
|
+
const match = /^https?:\/\/(?:www\.|mobile\.)?(?:x|twitter)\.com\/[A-Za-z0-9_]{1,15}\/status(?:es)?\/(\d+)/u.exec(String(url ?? "").trim());
|
|
36
|
+
return match ? match[1] : null;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const toIso = (value) => {
|
|
40
|
+
const date = new Date(value);
|
|
41
|
+
return Number.isNaN(date.getTime()) ? null : date.toISOString();
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
const entityLookup = (entityMap) => {
|
|
45
|
+
const entries = Array.isArray(entityMap)
|
|
46
|
+
? entityMap
|
|
47
|
+
: Object.entries(entityMap ?? {}).map(([key, value]) => ({ key, value }));
|
|
48
|
+
return new Map(entries.map((entry) => [String(entry.key), entry.value]));
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const blockLine = (block, entities) => {
|
|
52
|
+
const text = block.text ?? "";
|
|
53
|
+
switch (block.type) {
|
|
54
|
+
case "header-one": return `## ${text}`;
|
|
55
|
+
case "header-two": return `### ${text}`;
|
|
56
|
+
case "header-three": return `#### ${text}`;
|
|
57
|
+
case "unordered-list-item": return `- ${text}`;
|
|
58
|
+
case "ordered-list-item": return `1. ${text}`;
|
|
59
|
+
case "blockquote": return `> ${text}`;
|
|
60
|
+
case "code-block": return `\`\`\`\n${text}\n\`\`\``;
|
|
61
|
+
case "atomic": {
|
|
62
|
+
const entity = entities.get(String(block.entityRanges?.[0]?.key));
|
|
63
|
+
if (entity?.type === "MARKDOWN" && typeof entity.data?.markdown === "string") return entity.data.markdown;
|
|
64
|
+
if (entity?.type === "DIVIDER") return "---";
|
|
65
|
+
return "[media]";
|
|
66
|
+
}
|
|
67
|
+
default: return text;
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
// fxtwitter returns a post, and for a long-form X article also its Draft.js-style block list.
|
|
72
|
+
export const fxtwitterToMarkdown = (json) => {
|
|
73
|
+
const tweet = json?.tweet;
|
|
74
|
+
if (!tweet || typeof tweet !== "object") throw new Error("fxtwitter response carries no tweet");
|
|
75
|
+
const handle = tweet.author?.screen_name;
|
|
76
|
+
const author = handle ? `${tweet.author.name ?? handle} (@${handle})` : "unknown";
|
|
77
|
+
const published = toIso(tweet.created_at) ?? "unknown";
|
|
78
|
+
const blocks = tweet.article?.content?.blocks;
|
|
79
|
+
const lines = [];
|
|
80
|
+
let kind = "x-post";
|
|
81
|
+
let title = null;
|
|
82
|
+
if (Array.isArray(blocks)) {
|
|
83
|
+
kind = "x-article";
|
|
84
|
+
title = tweet.article.title ?? "Untitled article";
|
|
85
|
+
const entities = entityLookup(tweet.article.content.entityMap);
|
|
86
|
+
lines.push(`# ${title}`, "", `source: ${tweet.url}`, "");
|
|
87
|
+
for (const block of blocks) lines.push(blockLine(block, entities), "");
|
|
88
|
+
} else {
|
|
89
|
+
lines.push(`# Post by ${author}`, "", `source: ${tweet.url}`, "", tweet.text ?? "", "");
|
|
90
|
+
}
|
|
91
|
+
if (tweet.quote?.text) lines.push("## Quoted post", "", `source: ${tweet.quote.url ?? "unknown"}`, "", tweet.quote.text, "");
|
|
92
|
+
return { markdown: `${lines.join("\n").trimEnd()}\n`, meta: { kind, author, published, title, url: tweet.url ?? null, id: tweet.id ?? null } };
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
const decodeEntities = (text) => text
|
|
96
|
+
.replace(/ /gu, " ")
|
|
97
|
+
.replace(/&/gu, "&")
|
|
98
|
+
.replace(/</gu, "<")
|
|
99
|
+
.replace(/>/gu, ">")
|
|
100
|
+
.replace(/"/gu, "\"")
|
|
101
|
+
.replace(/'|'/gu, "'")
|
|
102
|
+
.replace(/&#(\d+);/gu, (_, code) => String.fromCodePoint(Number(code)));
|
|
103
|
+
|
|
104
|
+
export const htmlToText = (html) => decodeEntities(String(html)
|
|
105
|
+
.replace(/<(script|style|noscript|svg|nav|footer|header)\b[\s\S]*?<\/\1>/giu, " ")
|
|
106
|
+
.replace(/<h([1-6])\b[^>]*>/giu, (_, level) => `\n\n${"#".repeat(Number(level))} `)
|
|
107
|
+
.replace(/<li\b[^>]*>/giu, "\n- ")
|
|
108
|
+
.replace(/<(?:br|\/p|\/div|\/h[1-6]|\/li|\/pre|\/blockquote|\/tr)\b[^>]*>/giu, "\n")
|
|
109
|
+
.replace(/<[^>]+>/gu, " "))
|
|
110
|
+
.replace(/[ \t]+/gu, " ")
|
|
111
|
+
.replace(/\n\s*\n\s*\n+/gu, "\n\n")
|
|
112
|
+
.trim();
|
|
113
|
+
|
|
114
|
+
const metaContent = (html, pattern) => {
|
|
115
|
+
const match = pattern.exec(html);
|
|
116
|
+
return match ? decodeEntities(match[1]).trim() : null;
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
// ---------------------------------------------------------------------------------------------
|
|
120
|
+
// Licence table
|
|
121
|
+
|
|
122
|
+
const LICENCE_PATTERNS = [
|
|
123
|
+
["AGPL-3.0", /GNU AFFERO GENERAL PUBLIC LICENSE/iu],
|
|
124
|
+
["LGPL-3.0", /GNU LESSER GENERAL PUBLIC LICENSE[\s\S]{0,120}?Version 3/iu],
|
|
125
|
+
["LGPL-2.1", /GNU (?:LESSER|LIBRARY) GENERAL PUBLIC LICENSE/iu],
|
|
126
|
+
["GPL-3.0", /GNU GENERAL PUBLIC LICENSE[\s\S]{0,120}?Version 3/iu],
|
|
127
|
+
["GPL-2.0", /GNU GENERAL PUBLIC LICENSE[\s\S]{0,120}?Version 2/iu],
|
|
128
|
+
["MPL-2.0", /Mozilla Public License[\s\S]{0,40}?2\.0/iu],
|
|
129
|
+
["Apache-2.0", /Apache License[\s\S]{0,40}?Version 2\.0/iu],
|
|
130
|
+
["Unlicense", /free and unencumbered software released into the public domain/iu],
|
|
131
|
+
["CC0-1.0", /CC0 1\.0|Creative Commons Zero|creativecommons\.org\/publicdomain\/zero/iu],
|
|
132
|
+
["ISC", /Permission to use, copy, modify, and\/or distribute this software for any purpose/iu],
|
|
133
|
+
["MIT", /Permission is hereby granted, free of charge/iu],
|
|
134
|
+
];
|
|
135
|
+
|
|
136
|
+
const ccFromParts = (parts, version) => `CC-BY${parts.nc ? "-NC" : ""}${parts.sa ? "-SA" : ""}${parts.nd ? "-ND" : ""}-${version ?? "4.0"}`;
|
|
137
|
+
|
|
138
|
+
export const detectLicence = (text) => {
|
|
139
|
+
const body = String(text ?? "");
|
|
140
|
+
const spdxLine = /SPDX-License-Identifier:\s*([A-Za-z0-9.+-]+(?:\s+(?:AND|OR|WITH)\s+[A-Za-z0-9.+-]+)*)/u.exec(body);
|
|
141
|
+
if (spdxLine) return { spdx: spdxLine[1], evidence: spdxLine[0].trim() };
|
|
142
|
+
const packageField = /"license"\s*:\s*"([^"]+)"/u.exec(body);
|
|
143
|
+
if (packageField) return { spdx: packageField[1], evidence: packageField[0] };
|
|
144
|
+
const ccUrl = /creativecommons\.org\/licenses\/(by(?:-nc)?(?:-sa|-nd)?)\/(\d\.\d)/iu.exec(body);
|
|
145
|
+
if (ccUrl) return { spdx: `CC-${ccUrl[1].toUpperCase()}-${ccUrl[2]}`, evidence: ccUrl[0] };
|
|
146
|
+
const ccName = /creative\s+commons\s+attribution((?:[-\s]+(?:non-?commercial|share-?alike|no-?derivatives|no-?derivs))*)(?:\s+(\d\.\d))?/iu.exec(body);
|
|
147
|
+
if (ccName) {
|
|
148
|
+
const words = ccName[1].toLowerCase();
|
|
149
|
+
return {
|
|
150
|
+
spdx: ccFromParts({ nc: /non-?commercial/u.test(words), sa: /share-?alike/u.test(words), nd: /no-?deriv/u.test(words) }, ccName[2]),
|
|
151
|
+
evidence: ccName[0].trim(),
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
for (const [spdx, pattern] of LICENCE_PATTERNS) {
|
|
155
|
+
const match = pattern.exec(body);
|
|
156
|
+
if (match) return { spdx, evidence: match[0].replace(/\s+/gu, " ").slice(0, 100) };
|
|
157
|
+
}
|
|
158
|
+
if (/Redistribution and use in source and binary forms/iu.test(body)) {
|
|
159
|
+
return /Neither the name/iu.test(body)
|
|
160
|
+
? { spdx: "BSD-3-Clause", evidence: "BSD redistribution clause with the no-endorsement clause" }
|
|
161
|
+
: { spdx: "BSD-2-Clause", evidence: "BSD redistribution clause" };
|
|
162
|
+
}
|
|
163
|
+
return { spdx: "NOASSERTION", evidence: "no licence text found" };
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
const PROHIBITION = /\b(?:do not|don't|may not|must not)\s+(?:copy|reproduce|adapt|redistribute|reuse|republish|repost)\b|\bno part of this (?:work|publication|book|guide|document|course)\b|\b(?:confidential|internal use only|not for (?:re)?distribution|source[- ]available|proprietary)\b/iu;
|
|
167
|
+
// Reuse terms count only on lines about rights. An article that tells its readers "do not copy the
|
|
168
|
+
// reference's layout" is giving advice about its subject, not setting terms for itself.
|
|
169
|
+
const RIGHTS_LINE = /copyright|©|\(c\)\s*\d{4}|licen[cs]e|all rights|permission|terms of (?:use|service)|reproduc|redistribut|republish|derivative|confidential|proprietary|source[- ]available|no part of this/iu;
|
|
170
|
+
const rightsLines = (text) => String(text).split("\n").filter((line) => RIGHTS_LINE.test(line)).join("\n");
|
|
171
|
+
const PERMISSIVE = new Set(["MIT", "ISC", "BSD-2-Clause", "BSD-3-Clause", "Apache-2.0", "Zlib", "0BSD", "Unlicense", "CC0-1.0", "BSL-1.0", "Python-2.0"]);
|
|
172
|
+
|
|
173
|
+
// Ideas, methods and procedures are not protected expression, so every class may give up mechanisms.
|
|
174
|
+
// The class decides only whether the target ships a credit line and how strictly the build is fenced.
|
|
175
|
+
export const decideLicence = (spdx, { text } = {}) => {
|
|
176
|
+
const id = String(spdx ?? "NOASSERTION").trim();
|
|
177
|
+
const terms = text ? rightsLines(text) : "";
|
|
178
|
+
if (terms && PROHIBITION.test(terms)) {
|
|
179
|
+
return { decision: "needs-review", class: "restricted-terms", reason: `the source states reuse terms ("${PROHIBITION.exec(terms)[0]}"); a person decides ideas-only or no-go` };
|
|
180
|
+
}
|
|
181
|
+
if (PERMISSIVE.has(id)) return { decision: "ideas-credit", class: "permissive", reason: `${id} permits reuse; take mechanisms only and ship a credit line` };
|
|
182
|
+
if (/^CC-BY-(?:NC|ND)/u.test(id) || /^CC-BY-NC/u.test(id)) return { decision: "ideas-only", class: "restricted", reason: `${id} bars commercial use or sharing adaptations; take mechanisms only, never wording` };
|
|
183
|
+
if (/^CC-BY-SA/u.test(id) || /^(?:A|L)?GPL|^MPL|^EPL|^EUPL|^CDDL/u.test(id)) {
|
|
184
|
+
return { decision: "ideas-only", class: "copyleft", reason: `${id} is copyleft; take mechanisms only and keep the implementer away from the source` };
|
|
185
|
+
}
|
|
186
|
+
if (/^CC-BY-\d/u.test(id)) return { decision: "ideas-credit", class: "attribution", reason: `${id} asks for credit; take mechanisms only and ship a credit line` };
|
|
187
|
+
if (id === "NOASSERTION" || id === "") return { decision: "ideas-only", class: "unlicensed", reason: "no licence grant: all rights reserved, so take mechanisms only and credit in the report" };
|
|
188
|
+
return { decision: "ideas-only", class: "unrecognised", reason: `${id} is not in the table; treated as all rights reserved` };
|
|
189
|
+
};
|
|
190
|
+
|
|
191
|
+
// ---------------------------------------------------------------------------------------------
|
|
192
|
+
// Overlap gate
|
|
193
|
+
|
|
194
|
+
const WORD = /[\p{L}\p{N}]+/gu;
|
|
195
|
+
|
|
196
|
+
const wordsWithLines = (text) => {
|
|
197
|
+
const body = String(text ?? "").normalize("NFKC");
|
|
198
|
+
const out = [];
|
|
199
|
+
let line = 1;
|
|
200
|
+
let cursor = 0;
|
|
201
|
+
for (const match of body.matchAll(WORD)) {
|
|
202
|
+
for (let index = cursor; index < match.index; index += 1) if (body.charCodeAt(index) === 10) line += 1;
|
|
203
|
+
cursor = match.index;
|
|
204
|
+
out.push({ word: match[0].toLowerCase(), line });
|
|
205
|
+
}
|
|
206
|
+
return out;
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
export const findOverlap = ({ sources, targets, minWords = DEFAULT_MIN_WORDS }) => {
|
|
210
|
+
const size = Math.max(4, Number(minWords) || DEFAULT_MIN_WORDS);
|
|
211
|
+
const index = new Map();
|
|
212
|
+
for (const source of sources) {
|
|
213
|
+
const words = wordsWithLines(source.text).map((token) => token.word);
|
|
214
|
+
for (let start = 0; start + size <= words.length; start += 1) {
|
|
215
|
+
const key = words.slice(start, start + size).join(" ");
|
|
216
|
+
if (!index.has(key)) index.set(key, source.id);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
const runs = [];
|
|
220
|
+
for (const target of targets) {
|
|
221
|
+
const tokens = wordsWithLines(target.text);
|
|
222
|
+
const keyAt = (start) => tokens.slice(start, start + size).map((token) => token.word).join(" ");
|
|
223
|
+
let start = 0;
|
|
224
|
+
while (start + size <= tokens.length) {
|
|
225
|
+
const source = index.get(keyAt(start));
|
|
226
|
+
if (!source) {
|
|
227
|
+
start += 1;
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
let next = start + 1;
|
|
231
|
+
while (next + size <= tokens.length && index.has(keyAt(next))) next += 1;
|
|
232
|
+
const words = next - 1 - start + size;
|
|
233
|
+
runs.push({
|
|
234
|
+
path: target.path,
|
|
235
|
+
line: tokens[start].line,
|
|
236
|
+
words,
|
|
237
|
+
source,
|
|
238
|
+
excerpt: `${tokens.slice(start, start + 6).map((token) => token.word).join(" ")} …`,
|
|
239
|
+
});
|
|
240
|
+
start += words;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return runs;
|
|
244
|
+
};
|
|
245
|
+
|
|
246
|
+
const headingSet = (text) => new Set(String(text ?? "").split("\n")
|
|
247
|
+
.filter((line) => /^#{1,6}\s+\S/u.test(line))
|
|
248
|
+
.map((line) => wordsWithLines(line).map((token) => token.word).join(" "))
|
|
249
|
+
.filter((heading) => heading.split(" ").length >= 2));
|
|
250
|
+
|
|
251
|
+
const sharedHeadings = (sources, target) => {
|
|
252
|
+
const mine = headingSet(target.text);
|
|
253
|
+
return sources
|
|
254
|
+
.map((source) => ({ source: source.id, shared: [...headingSet(source.text)].filter((heading) => mine.has(heading)) }))
|
|
255
|
+
.filter((entry) => entry.shared.length >= 3);
|
|
256
|
+
};
|
|
257
|
+
|
|
258
|
+
// ---------------------------------------------------------------------------------------------
|
|
259
|
+
// Run folder
|
|
260
|
+
|
|
261
|
+
const sha256 = (buffer) => createHash("sha256").update(buffer).digest("hex");
|
|
262
|
+
const nowIso = () => new Date().toISOString();
|
|
263
|
+
const readJson = (path) => JSON.parse(readFileSync(path, "utf8"));
|
|
264
|
+
const writeJson = (path, value) => writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
|
|
265
|
+
const ledgerPath = (run) => join(run, "ledger.json");
|
|
266
|
+
const loadLedger = (run) => readJson(ledgerPath(run));
|
|
267
|
+
const fromRun = (run, path) => (isAbsolute(path) ? path : join(run, path));
|
|
268
|
+
const fromTarget = (ledger, path) => (isAbsolute(path) ? path : join(ledger.target?.root ?? "", path));
|
|
269
|
+
const filled = (value) => typeof value === "string" && value.trim().length > 0;
|
|
270
|
+
const takenMechanisms = (ledger) => (ledger.mechanisms ?? []).filter((mechanism) => ["take", "adapt"].includes(mechanism.port?.decision));
|
|
271
|
+
|
|
272
|
+
const isTextFile = (path) => {
|
|
273
|
+
if (!TEXT_EXTENSIONS.has(extname(path).toLowerCase())) return false;
|
|
274
|
+
const stats = statSync(path);
|
|
275
|
+
if (stats.size > 512 * 1024) return false;
|
|
276
|
+
return !readFileSync(path).subarray(0, 4096).includes(0);
|
|
277
|
+
};
|
|
278
|
+
|
|
279
|
+
const walkText = (dir, out = []) => {
|
|
280
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
281
|
+
if ([".git", "node_modules", "dist", "build", "vendor", ".venv"].includes(entry.name)) continue;
|
|
282
|
+
const path = join(dir, entry.name);
|
|
283
|
+
if (entry.isDirectory()) walkText(path, out);
|
|
284
|
+
else if (entry.isFile() && isTextFile(path)) out.push(path);
|
|
285
|
+
}
|
|
286
|
+
return out;
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
// Every captured text a run holds, so the overlap gate compares against all of it.
|
|
290
|
+
export const sourceTexts = (run, ledger = loadLedger(run)) => {
|
|
291
|
+
const texts = [];
|
|
292
|
+
for (const source of ledger.sources ?? []) {
|
|
293
|
+
for (const file of source.files ?? []) {
|
|
294
|
+
const path = fromRun(run, file.path);
|
|
295
|
+
if (!existsSync(path)) continue;
|
|
296
|
+
if (statSync(path).isDirectory()) {
|
|
297
|
+
for (const inner of walkText(path)) texts.push({ id: source.id, text: readFileSync(inner, "utf8") });
|
|
298
|
+
} else if (isTextFile(path) && !/raw\.(?:json|html)$/u.test(path)) {
|
|
299
|
+
texts.push({ id: source.id, text: readFileSync(path, "utf8") });
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return texts;
|
|
304
|
+
};
|
|
305
|
+
|
|
306
|
+
const lineCount = (text) => {
|
|
307
|
+
const lines = text.split("\n");
|
|
308
|
+
return text.endsWith("\n") ? lines.length - 1 : lines.length;
|
|
309
|
+
};
|
|
310
|
+
|
|
311
|
+
// A gap claim cites the target as it was before the port, so a ref resolves against the base
|
|
312
|
+
// revision when the target is a git checkout and against the working tree otherwise.
|
|
313
|
+
const baseText = (ledger, path) => {
|
|
314
|
+
const root = ledger.target?.root;
|
|
315
|
+
const base = ledger.target?.base_ref;
|
|
316
|
+
if (root && base && /^[0-9a-f]{7,40}$/iu.test(base)) {
|
|
317
|
+
const shown = spawnSync("git", ["-C", root, "show", `${base}:${path}`], { encoding: "utf8" });
|
|
318
|
+
if (shown.status === 0) return shown.stdout;
|
|
319
|
+
}
|
|
320
|
+
const full = fromTarget(ledger, path);
|
|
321
|
+
return existsSync(full) && statSync(full).isFile() ? readFileSync(full, "utf8") : null;
|
|
322
|
+
};
|
|
323
|
+
|
|
324
|
+
const sourceMentions = (ledger) => [
|
|
325
|
+
/(?:^|[\s(`'"/])sources\/S\d+/u,
|
|
326
|
+
...(ledger.sources ?? []).map((source) => source.url).filter(filled).map((url) => url.replace(/[?#].*$/u, "")),
|
|
327
|
+
];
|
|
328
|
+
|
|
329
|
+
const namesSources = (ledger, text) => sourceMentions(ledger).find((mention) => (
|
|
330
|
+
mention instanceof RegExp ? mention.test(text) : text.includes(mention)
|
|
331
|
+
));
|
|
332
|
+
|
|
333
|
+
const overlapIssues = (run, ledger, files, minWords) => {
|
|
334
|
+
const sources = sourceTexts(run, ledger);
|
|
335
|
+
const targets = files.map(({ label, path }) => ({ path: label, text: readFileSync(path, "utf8") }));
|
|
336
|
+
const issues = findOverlap({ sources, targets, minWords }).map((hit) => `${hit.path}:${hit.line} copies ${hit.words} words from ${hit.source} ("${hit.excerpt}"); rewrite it in your own words`);
|
|
337
|
+
const warnings = targets.flatMap((target) => sharedHeadings(sources, target).map((entry) => `${target.path} shares ${entry.shared.length} headings with ${entry.source}; check that the structure is your own`));
|
|
338
|
+
return { issues, warnings };
|
|
339
|
+
};
|
|
340
|
+
|
|
341
|
+
// lit-plan's checklist grammar: rows start at column zero under the exact `## TODOs` and
|
|
342
|
+
// `## Final verification` headings, numbered from 1 and from F1. Fenced examples do not count.
|
|
343
|
+
const TODO_ROW = /^- \[[ xX]\] (\d+)\. (\S.*?) — Action: (\S.*?); Output: (\S.*?); Verification: (\S.*)$/u;
|
|
344
|
+
const FINAL_ROW = /^- \[[ xX]\] F(\d+)\. (\S.*?) — Verification: (\S.*)$/u;
|
|
345
|
+
|
|
346
|
+
export const planGrammarIssues = (plan) => {
|
|
347
|
+
const todos = [];
|
|
348
|
+
const finals = [];
|
|
349
|
+
const issues = [];
|
|
350
|
+
let section = null;
|
|
351
|
+
let fenced = false;
|
|
352
|
+
for (const line of String(plan).split("\n")) {
|
|
353
|
+
if (/^\s*(?:```|~~~)/u.test(line)) {
|
|
354
|
+
fenced = !fenced;
|
|
355
|
+
continue;
|
|
356
|
+
}
|
|
357
|
+
if (fenced) continue;
|
|
358
|
+
const heading = /^## (.+?)\s*$/u.exec(line);
|
|
359
|
+
if (heading) {
|
|
360
|
+
section = heading[1] === "TODOs" ? todos : heading[1] === "Final verification" ? finals : null;
|
|
361
|
+
continue;
|
|
362
|
+
}
|
|
363
|
+
if (section === null || !/^- \[/u.test(line)) continue;
|
|
364
|
+
const row = (section === todos ? TODO_ROW : FINAL_ROW).exec(line);
|
|
365
|
+
if (!row) issues.push(`has a malformed row under ${section === todos ? "## TODOs" : "## Final verification"}: "${line.slice(0, 60)}"; write it as lit-plan's row grammar`);
|
|
366
|
+
else section.push(Number(row[1]));
|
|
367
|
+
}
|
|
368
|
+
if (todos.length === 0) issues.push("has no implementation row under ## TODOs (- [ ] 1. <title> — Action: <action>; Output: <output>; Verification: <verification>)");
|
|
369
|
+
if (finals.length === 0) issues.push("has no final-verifier row under ## Final verification (- [ ] F1. <title> — Verification: <verification>)");
|
|
370
|
+
for (const [label, numbers] of [["## TODOs", todos], ["## Final verification", finals]]) {
|
|
371
|
+
numbers.forEach((number, index) => {
|
|
372
|
+
if (number !== index + 1) issues.push(`numbers its ${label} rows ${numbers.join(", ")}; they must run 1, 2, 3 in order`);
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
return [...new Set(issues)];
|
|
376
|
+
};
|
|
377
|
+
|
|
378
|
+
const checkPhase = {
|
|
379
|
+
frame(run, ledger, issues) {
|
|
380
|
+
if (ledger.schema !== LEDGER_SCHEMA) issues.push(`ledger.json schema is not ${LEDGER_SCHEMA}`);
|
|
381
|
+
if (!["existing", "new"].includes(ledger.target?.kind)) issues.push("target.kind must be existing or new");
|
|
382
|
+
if (!filled(ledger.target?.root) || !existsSync(ledger.target.root)) issues.push("target.root is missing or does not exist");
|
|
383
|
+
if (ledger.target?.kind === "existing" && !filled(ledger.target?.base_ref)) issues.push("target.base_ref is missing: record the revision the gap analysis reads");
|
|
384
|
+
},
|
|
385
|
+
capture(run, ledger, issues) {
|
|
386
|
+
const sources = ledger.sources ?? [];
|
|
387
|
+
if (sources.length === 0) issues.push("no source captured");
|
|
388
|
+
const ids = new Set();
|
|
389
|
+
for (const source of sources) {
|
|
390
|
+
const id = source.id ?? "?";
|
|
391
|
+
if (!/^S\d+$/u.test(id) || ids.has(id)) issues.push(`${id}: id must be S<n> and unique`);
|
|
392
|
+
ids.add(id);
|
|
393
|
+
if (!KINDS.has(source.kind)) issues.push(`${id}: kind ${source.kind} is not one of ${[...KINDS].join(", ")}`);
|
|
394
|
+
if (!filled(source.url) && !filled(source.path)) issues.push(`${id}: url or path missing`);
|
|
395
|
+
for (const field of ["author", "published", "retrieved_at"]) if (!filled(source[field])) issues.push(`${id}: ${field} missing`);
|
|
396
|
+
if (!filled(source.licence?.spdx)) issues.push(`${id}: licence.spdx missing (use NOASSERTION when none is stated)`);
|
|
397
|
+
if (!Array.isArray(source.files) || source.files.length === 0) issues.push(`${id}: no captured file`);
|
|
398
|
+
for (const file of source.files ?? []) {
|
|
399
|
+
const path = fromRun(run, file.path ?? "");
|
|
400
|
+
if (!file.path || !existsSync(path)) {
|
|
401
|
+
issues.push(`${id}: ${file.path} is missing`);
|
|
402
|
+
} else if (file.commit) {
|
|
403
|
+
const head = spawnSync("git", ["-C", path, "rev-parse", "HEAD"], { encoding: "utf8" });
|
|
404
|
+
if (head.status !== 0 || head.stdout.trim() !== file.commit) issues.push(`${id}: ${file.path} is not at the captured commit ${file.commit}`);
|
|
405
|
+
} else if (statSync(path).isFile() && sha256(readFileSync(path)) !== file.sha256) {
|
|
406
|
+
issues.push(`${id}: ${file.path} sha256 mismatch; the capture changed after it was recorded`);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
},
|
|
411
|
+
licence(run, ledger, issues) {
|
|
412
|
+
for (const source of ledger.sources ?? []) {
|
|
413
|
+
const decision = source.licence?.decision;
|
|
414
|
+
if (decision === "needs-review") issues.push(`${source.id}: licence needs a person's decision (${source.licence.reason ?? "reuse terms found"}); record it with hijack.mjs licence --set`);
|
|
415
|
+
else if (!DECISIONS.has(decision)) issues.push(`${source.id}: licence.decision must be one of ${[...DECISIONS].join(", ")}`);
|
|
416
|
+
}
|
|
417
|
+
},
|
|
418
|
+
distil(run, ledger, issues) {
|
|
419
|
+
const mechanisms = ledger.mechanisms ?? [];
|
|
420
|
+
if (mechanisms.length === 0) issues.push("no mechanism distilled");
|
|
421
|
+
const sourceIds = new Set((ledger.sources ?? []).map((source) => source.id));
|
|
422
|
+
const ids = new Set();
|
|
423
|
+
for (const mechanism of mechanisms) {
|
|
424
|
+
const id = mechanism.id ?? "?";
|
|
425
|
+
if (!/^M\d+$/u.test(id) || ids.has(id)) issues.push(`${id}: id must be M<n> and unique`);
|
|
426
|
+
ids.add(id);
|
|
427
|
+
for (const field of ["title", "locator", "what", "why"]) if (!filled(mechanism[field])) issues.push(`${id}: ${field} missing`);
|
|
428
|
+
if (!EVIDENCE.has(mechanism.evidence)) issues.push(`${id}: evidence must be evidenced, anecdotal or opinion`);
|
|
429
|
+
if (!Array.isArray(mechanism.sources) || mechanism.sources.length === 0) issues.push(`${id}: names no source`);
|
|
430
|
+
for (const source of mechanism.sources ?? []) if (!sourceIds.has(source)) issues.push(`${id}: source ${source} is not in the ledger`);
|
|
431
|
+
}
|
|
432
|
+
},
|
|
433
|
+
gap(run, ledger, issues) {
|
|
434
|
+
const decisions = new Map((ledger.sources ?? []).map((source) => [source.id, source.licence?.decision]));
|
|
435
|
+
for (const mechanism of ledger.mechanisms ?? []) {
|
|
436
|
+
const id = mechanism.id;
|
|
437
|
+
const gap = mechanism.gap ?? {};
|
|
438
|
+
if (!GAP_STATUS.has(gap.status)) issues.push(`${id}: gap.status must be has, weak or lacks`);
|
|
439
|
+
const refs = Array.isArray(gap.refs) ? gap.refs : [];
|
|
440
|
+
if (ledger.target?.kind === "existing" && ["has", "weak"].includes(gap.status) && refs.length === 0) issues.push(`${id}: gap ${gap.status} needs at least one path:line ref`);
|
|
441
|
+
if (ledger.target?.kind === "existing" && gap.status === "lacks" && refs.length === 0 && !filled(gap.searched)) issues.push(`${id}: gap lacks needs gap.searched (what was searched and where)`);
|
|
442
|
+
for (const ref of refs) {
|
|
443
|
+
const match = /^(.+?):(\d+)(?:-(\d+))?$/u.exec(String(ref));
|
|
444
|
+
const text = match ? baseText(ledger, match[1]) : null;
|
|
445
|
+
const last = match ? Number(match[3] ?? match[2]) : 0;
|
|
446
|
+
if (!match || text === null || last < 1 || last > lineCount(text)) {
|
|
447
|
+
issues.push(`${id}: gap ref ${ref} does not resolve${text === null ? "" : ` (file has ${lineCount(text)} lines)`}`);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
const port = mechanism.port ?? {};
|
|
451
|
+
if (!PORT.has(port.decision)) issues.push(`${id}: port.decision must be take, adapt or leave`);
|
|
452
|
+
if (!filled(port.reason)) issues.push(`${id}: port.reason missing`);
|
|
453
|
+
if (["take", "adapt"].includes(port.decision)) {
|
|
454
|
+
const usable = (mechanism.sources ?? []).filter((source) => decisions.get(source) !== "no-go");
|
|
455
|
+
if (usable.length === 0) issues.push(`${id}: taken (${port.decision}) but every source it rests on is no-go`);
|
|
456
|
+
if (port.decision === "take" && gap.status === "has") issues.push(`${id}: the target already has it; use adapt or leave`);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
},
|
|
460
|
+
spec(run, ledger, issues, warnings) {
|
|
461
|
+
const taken = takenMechanisms(ledger);
|
|
462
|
+
if (taken.length === 0) {
|
|
463
|
+
warnings.push("no mechanism is taken; spec, build and parity have nothing to check");
|
|
464
|
+
return;
|
|
465
|
+
}
|
|
466
|
+
const specPath = join(run, "SPEC.md");
|
|
467
|
+
if (!existsSync(specPath)) {
|
|
468
|
+
issues.push("SPEC.md is missing");
|
|
469
|
+
return;
|
|
470
|
+
}
|
|
471
|
+
const spec = readFileSync(specPath, "utf8");
|
|
472
|
+
for (const mechanism of taken) {
|
|
473
|
+
if (!new RegExp(`^#{2,3} .*\\b${mechanism.id}\\b`, "mu").test(spec)) issues.push(`SPEC.md has no section for ${mechanism.id}`);
|
|
474
|
+
}
|
|
475
|
+
const named = namesSources(ledger, spec);
|
|
476
|
+
if (named) issues.push(`SPEC.md names the captured sources (${named instanceof RegExp ? "sources/S<n>" : named}); the implementer must never be pointed at them`);
|
|
477
|
+
const overlap = overlapIssues(run, ledger, [{ label: "SPEC.md", path: specPath }], ledger.min_words);
|
|
478
|
+
issues.push(...overlap.issues);
|
|
479
|
+
warnings.push(...overlap.warnings);
|
|
480
|
+
},
|
|
481
|
+
build(run, ledger, issues, warnings) {
|
|
482
|
+
if (takenMechanisms(ledger).length === 0) return;
|
|
483
|
+
const build = ledger.build ?? {};
|
|
484
|
+
// The plan uses lit-plan's row grammar (column-zero rows under `## TODOs` and `## Final verification`,
|
|
485
|
+
// the rows start-work parses) and cites every taken mechanism, so a reader can trace each SPEC
|
|
486
|
+
// section to the work that built it.
|
|
487
|
+
if (!filled(build.plan)) issues.push("build.plan is missing: record the plan lit-plan wrote from SPEC.md");
|
|
488
|
+
else {
|
|
489
|
+
const planPath = fromTarget(ledger, build.plan);
|
|
490
|
+
if (!existsSync(planPath)) issues.push(`build.plan ${build.plan} does not exist`);
|
|
491
|
+
else {
|
|
492
|
+
const plan = readFileSync(planPath, "utf8");
|
|
493
|
+
if (namesSources(ledger, plan)) issues.push(`build.plan ${build.plan} names the captured sources; plan rows cite SPEC sections only`);
|
|
494
|
+
for (const problem of planGrammarIssues(plan)) issues.push(`build.plan ${build.plan} ${problem}`);
|
|
495
|
+
for (const mechanism of takenMechanisms(ledger)) {
|
|
496
|
+
if (!new RegExp(`\\b${mechanism.id}\\b`, "u").test(plan)) issues.push(`build.plan ${build.plan} never cites ${mechanism.id}`);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
const changed = Array.isArray(build.changed) ? build.changed : [];
|
|
501
|
+
if (changed.length === 0) issues.push("build.changed lists no file");
|
|
502
|
+
const files = [];
|
|
503
|
+
for (const path of changed) {
|
|
504
|
+
const full = fromTarget(ledger, path);
|
|
505
|
+
if (!existsSync(full) || !statSync(full).isFile()) issues.push(`build.changed ${path} does not exist`);
|
|
506
|
+
else if (path !== ledger.attribution?.path) files.push({ label: path, path: full });
|
|
507
|
+
}
|
|
508
|
+
const overlap = overlapIssues(run, ledger, files, ledger.min_words);
|
|
509
|
+
issues.push(...overlap.issues);
|
|
510
|
+
warnings.push(...overlap.warnings);
|
|
511
|
+
},
|
|
512
|
+
parity(run, ledger, issues) {
|
|
513
|
+
const taken = takenMechanisms(ledger);
|
|
514
|
+
if (taken.length === 0) return;
|
|
515
|
+
for (const mechanism of taken) {
|
|
516
|
+
const parity = mechanism.parity ?? {};
|
|
517
|
+
for (const field of ["criterion", "red", "green", "surface", "artifact"]) if (!filled(parity[field])) issues.push(`${mechanism.id}: parity.${field} missing`);
|
|
518
|
+
if (filled(parity.artifact) && !existsSync(fromRun(run, parity.artifact))) issues.push(`${mechanism.id}: parity artifact ${parity.artifact} does not exist`);
|
|
519
|
+
}
|
|
520
|
+
const parity = ledger.parity ?? {};
|
|
521
|
+
const markers = Array.isArray(parity.markers) ? parity.markers.filter(filled) : [];
|
|
522
|
+
const read = (list, label) => (Array.isArray(list) ? list : []).flatMap((path) => {
|
|
523
|
+
const full = fromRun(run, path);
|
|
524
|
+
if (!existsSync(full)) {
|
|
525
|
+
issues.push(`${label} output ${path} does not exist`);
|
|
526
|
+
return [];
|
|
527
|
+
}
|
|
528
|
+
return [{ path, text: readFileSync(full, "utf8").toLowerCase() }];
|
|
529
|
+
});
|
|
530
|
+
const baseline = read(parity.baseline_outputs, "baseline");
|
|
531
|
+
const treatment = read(parity.treatment_outputs, "treatment");
|
|
532
|
+
if (markers.length === 0) issues.push("parity.markers is empty: name what the new behaviour leaves in its output");
|
|
533
|
+
if (baseline.length === 0) issues.push("parity.baseline_outputs is empty: run the old target on the same prompt");
|
|
534
|
+
if (treatment.length === 0) issues.push("parity.treatment_outputs is empty: run the new target on the same prompt");
|
|
535
|
+
for (const marker of markers) {
|
|
536
|
+
const needle = marker.toLowerCase();
|
|
537
|
+
const shown = treatment.filter((output) => output.text.includes(needle)).length;
|
|
538
|
+
if (treatment.length > 0 && shown === 0) issues.push(`marker "${marker}" never appears in a treatment output; the port did not change behaviour`);
|
|
539
|
+
else if (shown < treatment.length) issues.push(`marker "${marker}" appears in only ${shown} of ${treatment.length} treatment outputs; the behaviour is not reliable yet`);
|
|
540
|
+
for (const output of baseline) if (output.text.includes(needle)) issues.push(`baseline output ${output.path} shows marker "${marker}"; the baseline saw the new work or the gap was not real`);
|
|
541
|
+
}
|
|
542
|
+
},
|
|
543
|
+
report(run, ledger, issues) {
|
|
544
|
+
const reportPath = join(run, "REPORT.md");
|
|
545
|
+
if (!existsSync(reportPath)) {
|
|
546
|
+
issues.push("REPORT.md is missing");
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
const report = readFileSync(reportPath, "utf8");
|
|
550
|
+
for (const heading of ["## Taken", "## Left", "## Provenance"]) if (!report.includes(heading)) issues.push(`REPORT.md has no "${heading}" section`);
|
|
551
|
+
for (const mechanism of ledger.mechanisms ?? []) if (!new RegExp(`\\b${mechanism.id}\\b`, "u").test(report)) issues.push(`REPORT.md does not account for ${mechanism.id}`);
|
|
552
|
+
for (const source of ledger.sources ?? []) {
|
|
553
|
+
const handle = source.url ?? source.path;
|
|
554
|
+
if (filled(handle) && !report.includes(handle)) issues.push(`REPORT.md provenance does not list ${source.id} (${handle})`);
|
|
555
|
+
}
|
|
556
|
+
const takenSources = new Set(takenMechanisms(ledger).flatMap((mechanism) => mechanism.sources ?? []));
|
|
557
|
+
for (const source of ledger.sources ?? []) {
|
|
558
|
+
if (source.licence?.decision !== "ideas-credit" || !takenSources.has(source.id)) continue;
|
|
559
|
+
const notice = ledger.attribution?.path ? fromTarget(ledger, ledger.attribution.path) : null;
|
|
560
|
+
if (!notice || !existsSync(notice)) issues.push(`${source.id} needs credit: no attribution notice recorded (ledger.attribution.path)`);
|
|
561
|
+
else if (!readFileSync(notice, "utf8").includes(source.url ?? source.path ?? "")) issues.push(`${source.id} needs credit: ${ledger.attribution.path} does not name ${source.url ?? source.path}`);
|
|
562
|
+
}
|
|
563
|
+
},
|
|
564
|
+
};
|
|
565
|
+
|
|
566
|
+
export const checkRun = (run, { phase = "report" } = {}) => {
|
|
567
|
+
const issues = [];
|
|
568
|
+
const warnings = [];
|
|
569
|
+
if (!PHASES.includes(phase)) return { ok: false, phase, issues: [`unknown phase ${phase}; use one of ${PHASES.join(", ")}`], warnings };
|
|
570
|
+
if (!existsSync(ledgerPath(run))) return { ok: false, phase, issues: [`${ledgerPath(run)} does not exist; start with hijack.mjs init`], warnings };
|
|
571
|
+
const ledger = loadLedger(run);
|
|
572
|
+
for (const name of PHASES.slice(0, PHASES.indexOf(phase) + 1)) checkPhase[name](run, ledger, issues, warnings);
|
|
573
|
+
return { ok: issues.length === 0, phase, issues, warnings };
|
|
574
|
+
};
|
|
575
|
+
|
|
576
|
+
// The implementer sees the spec and the target, never the run folder, the sources or their URLs.
|
|
577
|
+
export const buildImplementerBrief = (run) => {
|
|
578
|
+
const gate = checkRun(run, { phase: "spec" });
|
|
579
|
+
if (!gate.ok) throw new Error(`the spec is not ready for an implementer:\n- ${gate.issues.join("\n- ")}`);
|
|
580
|
+
const ledger = loadLedger(run);
|
|
581
|
+
const spec = readFileSync(join(run, "SPEC.md"), "utf8").trim();
|
|
582
|
+
const taken = takenMechanisms(ledger).map((mechanism) => mechanism.id).join(", ");
|
|
583
|
+
return [
|
|
584
|
+
`TASK: Implement the specification below in the target at ${ledger.target.root}. You are the implementer of a clean-room port: work only from this specification and the target's own files. Sections to build: ${taken}.`,
|
|
585
|
+
"DELIVERABLE: the change each SPEC section asks for, written in the target's own naming, idiom, tests and manifest; the tests each section names; and the list of changed files as paths relative to the target root.",
|
|
586
|
+
"SCOPE: write only inside the target. Do not open any .litopencode/hijack/ folder, do not search for or fetch the original material the spec came from, and do not reconstruct it from memory. If a section is ambiguous, stop and return the question instead of guessing.",
|
|
587
|
+
"VERIFY: run the checks each section names and report their results. Return the changed file list so the run's build gate can compare it against the captured sources.",
|
|
588
|
+
"",
|
|
589
|
+
"--- SPEC ---",
|
|
590
|
+
spec,
|
|
591
|
+
].join("\n");
|
|
592
|
+
};
|
|
593
|
+
|
|
594
|
+
// ---------------------------------------------------------------------------------------------
|
|
595
|
+
// Capture
|
|
596
|
+
|
|
597
|
+
// Every hop is checked before it connects: http or https only, a public address (a name must resolve
|
|
598
|
+
// to public addresses only), at most five redirects, and the body is cut off once it passes the cap.
|
|
599
|
+
// The connection goes to the address the check approved, so a name that answers again with a private
|
|
600
|
+
// address (DNS rebinding) cannot move it; TLS still verifies the certificate for the name.
|
|
601
|
+
const isPublicAddress = (raw) => {
|
|
602
|
+
const address = raw.replace(/^\[|\]$/gu, "").toLowerCase();
|
|
603
|
+
if (address.startsWith("::ffff:")) return isPublicAddress(address.slice(7));
|
|
604
|
+
const version = isIP(address);
|
|
605
|
+
if (version === 4) {
|
|
606
|
+
const [a, b] = address.split(".").map(Number);
|
|
607
|
+
return !(a === 0 || a === 10 || a === 127 || (a === 100 && b >= 64 && b <= 127) || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || (a === 198 && (b === 18 || b === 19)) || a >= 224);
|
|
608
|
+
}
|
|
609
|
+
if (version === 6) return !(address === "::" || address === "::1" || /^(?:f[cd]|fe[89ab]|ff)/u.test(address));
|
|
610
|
+
return false;
|
|
611
|
+
};
|
|
612
|
+
|
|
613
|
+
const assertPublicHost = async (url, lookup) => {
|
|
614
|
+
const host = url.hostname.replace(/^\[|\]$/gu, "").toLowerCase();
|
|
615
|
+
if (host === "localhost" || host.endsWith(".localhost")) throw new Error(`${url.href} points at localhost`);
|
|
616
|
+
if (/^\d+$/u.test(host) || host.split(".").some((part) => /^0x[0-9a-f]+$|^0\d+$/iu.test(part))) throw new Error(`${url.href} uses a non-canonical numeric host`);
|
|
617
|
+
if (isIP(host)) {
|
|
618
|
+
if (!isPublicAddress(host)) throw new Error(`${url.href} is not a public address (private, loopback or link-local)`);
|
|
619
|
+
return { address: host, family: isIP(host) };
|
|
620
|
+
}
|
|
621
|
+
const records = await lookup(host, { all: true, verbatim: true });
|
|
622
|
+
if (records.length === 0 || records.some((record) => !isPublicAddress(record.address))) throw new Error(`${url.href} does not resolve to a public address only (private, loopback or link-local)`);
|
|
623
|
+
return { address: records[0].address, family: records[0].family };
|
|
624
|
+
};
|
|
625
|
+
|
|
626
|
+
// node:http(s) with a lookup that answers only with the approved address, read back as a Response.
|
|
627
|
+
const pinnedSend = (request) => (href, { headers, signal }, { address, family }) => new Promise((resolveResponse, reject) => {
|
|
628
|
+
const pin = (_host, options, callback) => (options?.all ? callback(null, [{ address, family }]) : callback(null, address, family));
|
|
629
|
+
const req = request(href, { headers, signal, lookup: pin }, (res) => {
|
|
630
|
+
const answer = new Headers();
|
|
631
|
+
for (const [name, value] of Object.entries(res.headers)) for (const item of [value].flat()) if (item !== undefined) answer.append(name, String(item));
|
|
632
|
+
const empty = [204, 205, 304].includes(res.statusCode);
|
|
633
|
+
if (empty) res.resume();
|
|
634
|
+
try {
|
|
635
|
+
resolveResponse(new Response(empty ? null : Readable.toWeb(res), { status: res.statusCode, headers: answer }));
|
|
636
|
+
} catch (error) {
|
|
637
|
+
res.destroy();
|
|
638
|
+
reject(error);
|
|
639
|
+
}
|
|
640
|
+
});
|
|
641
|
+
req.on("error", reject);
|
|
642
|
+
req.end();
|
|
643
|
+
});
|
|
644
|
+
|
|
645
|
+
const readCapped = async (response, limit, url) => {
|
|
646
|
+
const declared = Number(response.headers.get("content-length") ?? 0);
|
|
647
|
+
if (declared > limit) throw new Error(`${url} is larger than ${limit} bytes`);
|
|
648
|
+
if (!response.body) return Buffer.alloc(0);
|
|
649
|
+
const chunks = [];
|
|
650
|
+
let total = 0;
|
|
651
|
+
const reader = response.body.getReader();
|
|
652
|
+
for (;;) {
|
|
653
|
+
const { done, value } = await reader.read();
|
|
654
|
+
if (done) break;
|
|
655
|
+
total += value.byteLength;
|
|
656
|
+
if (total > limit) {
|
|
657
|
+
await reader.cancel().catch(() => undefined);
|
|
658
|
+
throw new Error(`${url} is larger than ${limit} bytes`);
|
|
659
|
+
}
|
|
660
|
+
chunks.push(Buffer.from(value));
|
|
661
|
+
}
|
|
662
|
+
return Buffer.concat(chunks);
|
|
663
|
+
};
|
|
664
|
+
|
|
665
|
+
// `fetchImpl` (a fetch-shaped stub, not pinned), `request`/`httpRequest` (node's) and `lookup` are injectable for tests.
|
|
666
|
+
export const publicFetch = async (url, { fetchImpl = null, request = httpsRequest, httpRequest: plainRequest = httpRequest, lookup = dnsLookup, limit = FETCH_LIMIT, timeoutMs = 30_000, maxRedirects = 5 } = {}) => {
|
|
667
|
+
const send = fetchImpl ? (href, init) => fetchImpl(href, init) : (href, init, pin) => pinnedSend(new URL(href).protocol === "http:" ? plainRequest : request)(href, init, pin);
|
|
668
|
+
let current = new URL(url);
|
|
669
|
+
const signal = AbortSignal.timeout(timeoutMs);
|
|
670
|
+
for (let hop = 0; ; hop += 1) {
|
|
671
|
+
if (current.protocol !== "https:" && current.protocol !== "http:") throw new Error(`${current.href}: only http and https links are captured`);
|
|
672
|
+
const pin = await assertPublicHost(current, lookup);
|
|
673
|
+
const response = await send(current.href, { headers: { "user-agent": "litopencode-lit-hijack/1 (+capture)" }, redirect: "manual", signal }, pin);
|
|
674
|
+
const location = response.status >= 300 && response.status < 400 ? response.headers.get("location") : null;
|
|
675
|
+
if (location) {
|
|
676
|
+
if (hop >= maxRedirects) throw new Error(`${url} redirected more than ${maxRedirects} times`);
|
|
677
|
+
await response.body?.cancel().catch(() => undefined);
|
|
678
|
+
current = new URL(location, current);
|
|
679
|
+
continue;
|
|
680
|
+
}
|
|
681
|
+
if (!response.ok) throw new Error(`${current.href} answered HTTP ${response.status}`);
|
|
682
|
+
return { buffer: await readCapped(response, limit, current.href), type: response.headers.get("content-type") ?? "", finalUrl: current.href };
|
|
683
|
+
}
|
|
684
|
+
};
|
|
685
|
+
|
|
686
|
+
const fetchBytes = (url) => publicFetch(url);
|
|
687
|
+
|
|
688
|
+
// A tree URL's folder must stay inside the clone; anything that climbs out reads as no folder.
|
|
689
|
+
export const containedSubpath = (repoDir, subpath) => {
|
|
690
|
+
const resolved = resolve(repoDir, `.${subpath}`);
|
|
691
|
+
return resolved === repoDir || resolved.startsWith(`${repoDir}${sep}`) ? resolved : null;
|
|
692
|
+
};
|
|
693
|
+
|
|
694
|
+
// A ref from a URL goes to git as a plain branch or tag name, never as something that reads as an option.
|
|
695
|
+
export const safeGitRef = (ref) => (typeof ref === "string" && /^[A-Za-z0-9._/][A-Za-z0-9._/-]{0,127}$/u.test(ref) && !ref.includes("..") ? ref : null);
|
|
696
|
+
|
|
697
|
+
const guessKind = (input) => {
|
|
698
|
+
if (parseXStatusUrl(input)) return "x";
|
|
699
|
+
if (/^https?:\/\/gist\.github\.com\//iu.test(input)) return "gist";
|
|
700
|
+
if (/^https?:\/\/github\.com\/[^/]+\/[^/]+/iu.test(input)) return "github-repo";
|
|
701
|
+
if (/\.pdf(?:[?#].*)?$/iu.test(input)) return "pdf";
|
|
702
|
+
if (/^https?:\/\//iu.test(input)) return "web";
|
|
703
|
+
return "file";
|
|
704
|
+
};
|
|
705
|
+
|
|
706
|
+
const git = (args, cwd) => {
|
|
707
|
+
const result = spawnSync("git", args, { cwd, encoding: "utf8", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } });
|
|
708
|
+
if (result.status !== 0) throw new Error(`git ${args.join(" ")} failed: ${result.stderr.trim()}`);
|
|
709
|
+
return result.stdout.trim();
|
|
710
|
+
};
|
|
711
|
+
|
|
712
|
+
const shallowClone = (url, dest, ref) => {
|
|
713
|
+
git(["-c", "core.hooksPath=/dev/null", "clone", "--depth", "1", "--no-tags", "--single-branch", ...(ref ? ["--branch", ref] : []), url, dest]);
|
|
714
|
+
return git(["rev-parse", "HEAD"], dest);
|
|
715
|
+
};
|
|
716
|
+
|
|
717
|
+
const repoLicence = (dir) => {
|
|
718
|
+
const candidates = readdirSync(dir).filter((name) => /^(?:licen[cs]e|copying|unlicense)(?:\.[a-z]+)?$|^package\.json$/iu.test(name));
|
|
719
|
+
const packageLast = (name) => Number(/package\.json/u.test(name));
|
|
720
|
+
for (const name of candidates.sort((left, right) => packageLast(left) - packageLast(right))) {
|
|
721
|
+
const found = detectLicence(readFileSync(join(dir, name), "utf8"));
|
|
722
|
+
if (found.spdx !== "NOASSERTION") return { ...found, evidence: `${name}: ${found.evidence}` };
|
|
723
|
+
}
|
|
724
|
+
return { spdx: "NOASSERTION", evidence: "no LICENSE, COPYING or package.json licence at the repository root" };
|
|
725
|
+
};
|
|
726
|
+
|
|
727
|
+
const fileEntry = (run, path) => {
|
|
728
|
+
const buffer = readFileSync(path);
|
|
729
|
+
return { path: relative(run, path), sha256: sha256(buffer), bytes: buffer.length };
|
|
730
|
+
};
|
|
731
|
+
|
|
732
|
+
export const captureSource = async (run, input, options = {}) => {
|
|
733
|
+
const ledger = loadLedger(run);
|
|
734
|
+
ledger.sources ??= [];
|
|
735
|
+
const id = `S${ledger.sources.length + 1}`;
|
|
736
|
+
const dir = join(run, "sources", id);
|
|
737
|
+
mkdirSync(dir, { recursive: true });
|
|
738
|
+
const kind = options.kind ?? guessKind(input);
|
|
739
|
+
const entry = { id, kind, url: options.url ?? (/^https?:\/\//iu.test(input) ? input : null), path: null, author: null, published: null, title: options.title ?? null, retrieved_at: nowIso(), files: [], licence: null, inert: true };
|
|
740
|
+
let licence = { spdx: "NOASSERTION", evidence: "no licence stated" };
|
|
741
|
+
|
|
742
|
+
if (kind === "x" || kind === "x-post" || kind === "x-article") {
|
|
743
|
+
const statusId = parseXStatusUrl(input);
|
|
744
|
+
if (!statusId) throw new Error(`${input} is not an x.com or twitter.com status URL`);
|
|
745
|
+
const { buffer } = await fetchBytes(`https://api.fxtwitter.com/status/${statusId}`);
|
|
746
|
+
const json = JSON.parse(buffer.toString("utf8"));
|
|
747
|
+
if (json.code !== 200) throw new Error(`fxtwitter answered code ${json.code} (${json.message ?? "no message"}) for ${input}`);
|
|
748
|
+
const { markdown, meta } = fxtwitterToMarkdown(json);
|
|
749
|
+
writeFileSync(join(dir, "raw.json"), buffer);
|
|
750
|
+
writeFileSync(join(dir, "content.md"), markdown);
|
|
751
|
+
Object.assign(entry, { kind: meta.kind, url: meta.url ?? input, author: meta.author, published: meta.published, title: entry.title ?? meta.title });
|
|
752
|
+
licence = { spdx: "NOASSERTION", evidence: "X post: the author keeps the rights; the platform's licence runs to the platform, not to readers" };
|
|
753
|
+
entry.files.push(fileEntry(run, join(dir, "content.md")), fileEntry(run, join(dir, "raw.json")));
|
|
754
|
+
} else if (kind === "github-repo" || kind === "gist") {
|
|
755
|
+
const match = /^https?:\/\/(?:gist\.)?github\.com\/([^/]+)\/([^/#?]+)(?:\/tree\/([^/]+)((?:\/[^#?]+)?))?/iu.exec(input);
|
|
756
|
+
if (!match) throw new Error(`${input} is not a github.com repository or gist URL`);
|
|
757
|
+
const cloneUrl = kind === "gist" ? `https://gist.github.com/${match[2].replace(/\.git$/u, "")}.git` : `https://github.com/${match[1]}/${match[2].replace(/\.git$/u, "")}.git`;
|
|
758
|
+
const repoDir = join(dir, "repo");
|
|
759
|
+
const ref = kind === "gist" || !match[3] ? null : safeGitRef(match[3]);
|
|
760
|
+
if (match[3] && kind !== "gist" && !ref) throw new Error(`${input} names a ref that is not a plain branch or tag name`);
|
|
761
|
+
const subpath = match[4] ? containedSubpath(repoDir, match[4]) : null;
|
|
762
|
+
if (match[4] && !subpath) throw new Error(`${input} names a folder outside the repository`);
|
|
763
|
+
const commit = shallowClone(cloneUrl, repoDir, ref);
|
|
764
|
+
Object.assign(entry, { author: match[1], published: git(["log", "-1", "--format=%cI"], repoDir), title: entry.title ?? `${match[1]}/${match[2]}` });
|
|
765
|
+
// A licence can differ per folder (one skill in a repo may be source-available), so the folder
|
|
766
|
+
// the URL points at is read before the repository root.
|
|
767
|
+
const folder = subpath;
|
|
768
|
+
const folderLicence = folder && existsSync(folder) && statSync(folder).isDirectory() ? repoLicence(folder) : null;
|
|
769
|
+
licence = kind === "gist"
|
|
770
|
+
? { spdx: "NOASSERTION", evidence: "gists carry no licence unless their files state one" }
|
|
771
|
+
: folderLicence && folderLicence.spdx !== "NOASSERTION" ? { ...folderLicence, evidence: `${relative(repoDir, subpath)}/${folderLicence.evidence}` } : repoLicence(repoDir);
|
|
772
|
+
if (subpath && subpath !== repoDir) entry.subpath = relative(repoDir, subpath);
|
|
773
|
+
entry.files.push({ path: relative(run, repoDir), commit });
|
|
774
|
+
} else if (kind === "web") {
|
|
775
|
+
const { buffer, type, finalUrl } = await fetchBytes(input);
|
|
776
|
+
if (/pdf/iu.test(type) || buffer.subarray(0, 5).toString("latin1") === "%PDF-") return captureSource(run, input, { ...options, kind: "pdf" });
|
|
777
|
+
const html = buffer.toString("utf8");
|
|
778
|
+
writeFileSync(join(dir, "raw.html"), buffer);
|
|
779
|
+
const title = metaContent(html, /<title[^>]*>([\s\S]*?)<\/title>/iu);
|
|
780
|
+
writeFileSync(join(dir, "content.md"), `# ${title ?? input}\n\nsource: ${finalUrl || input}\n\n${htmlToText(html)}\n`);
|
|
781
|
+
Object.assign(entry, {
|
|
782
|
+
url: finalUrl || input,
|
|
783
|
+
author: metaContent(html, /<meta[^>]+(?:name|property)=["'](?:author|article:author)["'][^>]+content=["']([^"']+)/iu),
|
|
784
|
+
published: metaContent(html, /<meta[^>]+(?:property|name)=["'](?:article:published_time|date|dc\.date)["'][^>]+content=["']([^"']+)/iu),
|
|
785
|
+
title: entry.title ?? title,
|
|
786
|
+
});
|
|
787
|
+
const licenceLink = /<a[^>]+rel=["']license["'][^>]+href=["']([^"']+)/iu.exec(html) ?? /<link[^>]+rel=["']license["'][^>]+href=["']([^"']+)/iu.exec(html);
|
|
788
|
+
if (licenceLink) licence = { ...detectLicence(licenceLink[1]), evidence: `rel=license ${licenceLink[1]}` };
|
|
789
|
+
entry.files.push(fileEntry(run, join(dir, "content.md")), fileEntry(run, join(dir, "raw.html")));
|
|
790
|
+
} else if (kind === "pdf") {
|
|
791
|
+
const buffer = /^https?:\/\//iu.test(input) ? (await fetchBytes(input)).buffer : readFileSync(input);
|
|
792
|
+
if (buffer.subarray(0, 5).toString("latin1") !== "%PDF-") throw new Error(`${input} is not a PDF: its bytes do not start with %PDF-`);
|
|
793
|
+
const pdfPath = join(dir, "source.pdf");
|
|
794
|
+
writeFileSync(pdfPath, buffer);
|
|
795
|
+
entry.files.push(fileEntry(run, pdfPath));
|
|
796
|
+
const text = spawnSync("pdftotext", ["-layout", pdfPath, join(dir, "content.md")], { encoding: "utf8" });
|
|
797
|
+
if (text.status === 0 && existsSync(join(dir, "content.md"))) {
|
|
798
|
+
entry.files.push(fileEntry(run, join(dir, "content.md")));
|
|
799
|
+
licence = detectLicence(readFileSync(join(dir, "content.md"), "utf8"));
|
|
800
|
+
} else {
|
|
801
|
+
entry.text_status = "pdftotext unavailable or failed; read source.pdf directly and record the mechanisms by page";
|
|
802
|
+
}
|
|
803
|
+
if (!/^https?:\/\//iu.test(input)) entry.path = resolve(input);
|
|
804
|
+
} else if (kind === "video") {
|
|
805
|
+
if (!options.transcript) throw new Error("a video capture needs --transcript <file>; save the captions first");
|
|
806
|
+
if (!entry.url) throw new Error("a video capture needs the video URL as input or --url");
|
|
807
|
+
copyFileSync(options.transcript, join(dir, "content.md"));
|
|
808
|
+
entry.files.push(fileEntry(run, join(dir, "content.md")));
|
|
809
|
+
} else if (kind === "file") {
|
|
810
|
+
const target = join(dir, `content${extname(input) || ".md"}`);
|
|
811
|
+
copyFileSync(input, target);
|
|
812
|
+
entry.path = resolve(input);
|
|
813
|
+
entry.files.push(fileEntry(run, target));
|
|
814
|
+
licence = detectLicence(readFileSync(target, "utf8"));
|
|
815
|
+
} else {
|
|
816
|
+
throw new Error(`unknown kind ${kind}`);
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
entry.author = options.author ?? entry.author ?? "unknown";
|
|
820
|
+
entry.published = options.date ?? entry.published ?? "unknown";
|
|
821
|
+
entry.licence = options.licence
|
|
822
|
+
? { spdx: options.licence, evidence: options.licenceEvidence ?? "stated by the user at capture", decision: null }
|
|
823
|
+
: { ...licence, decision: null };
|
|
824
|
+
ledger.sources.push(entry);
|
|
825
|
+
writeJson(ledgerPath(run), ledger);
|
|
826
|
+
return entry;
|
|
827
|
+
};
|
|
828
|
+
|
|
829
|
+
export const applyLicenceTable = (run, overrides = {}) => {
|
|
830
|
+
const ledger = loadLedger(run);
|
|
831
|
+
for (const source of ledger.sources ?? []) {
|
|
832
|
+
const override = overrides[source.id];
|
|
833
|
+
if (override) {
|
|
834
|
+
source.licence = { ...source.licence, decision: override.decision, decided_by: "user", reason: override.reason ?? "decided by the user" };
|
|
835
|
+
continue;
|
|
836
|
+
}
|
|
837
|
+
if (source.licence?.decided_by === "user") continue;
|
|
838
|
+
const text = sourceTexts(run, { sources: [source] }).map((entry) => entry.text).join("\n").slice(0, 200_000);
|
|
839
|
+
const verdict = decideLicence(source.licence?.spdx, { text });
|
|
840
|
+
source.licence = { ...source.licence, ...verdict, decided_by: "rule" };
|
|
841
|
+
}
|
|
842
|
+
writeJson(ledgerPath(run), ledger);
|
|
843
|
+
return ledger.sources ?? [];
|
|
844
|
+
};
|
|
845
|
+
|
|
846
|
+
export const initRun = (run, { target, kind = "existing", baseRef, title } = {}) => {
|
|
847
|
+
if (!target) throw new Error("init needs --target <project root>");
|
|
848
|
+
const root = resolve(target);
|
|
849
|
+
if (!existsSync(root)) throw new Error(`target ${root} does not exist`);
|
|
850
|
+
if (existsSync(ledgerPath(run))) throw new Error(`${ledgerPath(run)} already exists; resume that run instead`);
|
|
851
|
+
let base = baseRef ?? null;
|
|
852
|
+
if (!base && kind === "existing") {
|
|
853
|
+
const head = spawnSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" });
|
|
854
|
+
base = head.status === 0 ? head.stdout.trim() : null;
|
|
855
|
+
}
|
|
856
|
+
mkdirSync(join(run, "sources"), { recursive: true });
|
|
857
|
+
mkdirSync(join(run, "parity"), { recursive: true });
|
|
858
|
+
const ledger = {
|
|
859
|
+
schema: LEDGER_SCHEMA,
|
|
860
|
+
title: title ?? null,
|
|
861
|
+
created_at: nowIso(),
|
|
862
|
+
target: { kind, root, base_ref: base },
|
|
863
|
+
min_words: DEFAULT_MIN_WORDS,
|
|
864
|
+
sources: [],
|
|
865
|
+
mechanisms: [],
|
|
866
|
+
build: {},
|
|
867
|
+
parity: {},
|
|
868
|
+
};
|
|
869
|
+
writeJson(ledgerPath(run), ledger);
|
|
870
|
+
return ledger;
|
|
871
|
+
};
|
|
872
|
+
|
|
873
|
+
// ---------------------------------------------------------------------------------------------
|
|
874
|
+
// CLI
|
|
875
|
+
|
|
876
|
+
const USAGE = `Usage: hijack.mjs <command> <run-dir> [options]
|
|
877
|
+
init <run> --target <root> [--new] [--base-ref <rev>] [--title <text>]
|
|
878
|
+
capture <run> <url|file> [--kind x|github-repo|gist|web|pdf|video|file] [--url <u>] [--author <a>] [--date <d>] [--title <t>] [--licence <SPDX>] [--transcript <file>]
|
|
879
|
+
licence <run> [--set S1=ideas-only|ideas-credit|no-go --reason <text>]
|
|
880
|
+
overlap <run> [paths...] [--min-words <n>] [--json]
|
|
881
|
+
brief <run>
|
|
882
|
+
check <run> [--phase ${PHASES.join("|")}] [--json]`;
|
|
883
|
+
|
|
884
|
+
const parseArgs = (argv) => {
|
|
885
|
+
const positional = [];
|
|
886
|
+
const flags = {};
|
|
887
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
888
|
+
const arg = argv[index];
|
|
889
|
+
if (!arg.startsWith("--")) {
|
|
890
|
+
positional.push(arg);
|
|
891
|
+
continue;
|
|
892
|
+
}
|
|
893
|
+
const name = arg.slice(2);
|
|
894
|
+
if (["new", "json"].includes(name)) flags[name] = true;
|
|
895
|
+
else if (name === "set") (flags.set ??= []).push(argv[++index]);
|
|
896
|
+
else flags[name] = argv[++index];
|
|
897
|
+
}
|
|
898
|
+
return { positional, flags };
|
|
899
|
+
};
|
|
900
|
+
|
|
901
|
+
const main = async (argv) => {
|
|
902
|
+
const [command, ...rest] = argv;
|
|
903
|
+
const { positional, flags } = parseArgs(rest);
|
|
904
|
+
const run = positional[0] ? resolve(positional[0]) : null;
|
|
905
|
+
if (!command || !run) {
|
|
906
|
+
console.error(USAGE);
|
|
907
|
+
return 64;
|
|
908
|
+
}
|
|
909
|
+
if (command === "init") {
|
|
910
|
+
const ledger = initRun(run, { target: flags.target, kind: flags.new ? "new" : "existing", baseRef: flags["base-ref"], title: flags.title });
|
|
911
|
+
console.log(`run ${run}\ntarget ${ledger.target.root} (${ledger.target.kind}, base ${ledger.target.base_ref ?? "none: snapshot the target before the build"})`);
|
|
912
|
+
return 0;
|
|
913
|
+
}
|
|
914
|
+
if (command === "capture") {
|
|
915
|
+
if (!positional[1]) {
|
|
916
|
+
console.error("capture needs a URL or a file");
|
|
917
|
+
return 64;
|
|
918
|
+
}
|
|
919
|
+
const entry = await captureSource(run, positional[1], {
|
|
920
|
+
kind: flags.kind, url: flags.url, author: flags.author, date: flags.date, title: flags.title,
|
|
921
|
+
licence: flags.licence, licenceEvidence: flags["licence-evidence"], transcript: flags.transcript,
|
|
922
|
+
});
|
|
923
|
+
console.log(`${entry.id} ${entry.kind} ${entry.url ?? entry.path}\n author ${entry.author} · published ${entry.published} · licence ${entry.licence.spdx}\n files ${entry.files.map((file) => file.path).join(", ")}${entry.text_status ? `\n note ${entry.text_status}` : ""}`);
|
|
924
|
+
return 0;
|
|
925
|
+
}
|
|
926
|
+
if (command === "licence") {
|
|
927
|
+
const overrides = {};
|
|
928
|
+
for (const pair of flags.set ?? []) {
|
|
929
|
+
const [id, decision] = String(pair).split("=");
|
|
930
|
+
if (!DECISIONS.has(decision)) throw new Error(`--set ${pair}: decision must be one of ${[...DECISIONS].join(", ")}`);
|
|
931
|
+
overrides[id] = { decision, reason: flags.reason };
|
|
932
|
+
}
|
|
933
|
+
for (const source of applyLicenceTable(run, overrides)) {
|
|
934
|
+
console.log(`${source.id} ${source.licence.spdx} -> ${source.licence.decision} (${source.licence.decided_by}): ${source.licence.reason}`);
|
|
935
|
+
}
|
|
936
|
+
return 0;
|
|
937
|
+
}
|
|
938
|
+
if (command === "overlap") {
|
|
939
|
+
const ledger = loadLedger(run);
|
|
940
|
+
const paths = positional.slice(1);
|
|
941
|
+
const files = paths.length > 0
|
|
942
|
+
? paths.map((path) => ({ label: path, path: resolve(path) }))
|
|
943
|
+
: [
|
|
944
|
+
...(existsSync(join(run, "SPEC.md")) ? [{ label: "SPEC.md", path: join(run, "SPEC.md") }] : []),
|
|
945
|
+
...(ledger.build?.changed ?? []).map((path) => ({ label: path, path: fromTarget(ledger, path) })).filter((file) => existsSync(file.path)),
|
|
946
|
+
];
|
|
947
|
+
const minWords = Number(flags["min-words"] ?? ledger.min_words ?? DEFAULT_MIN_WORDS);
|
|
948
|
+
const { issues, warnings } = overlapIssues(run, ledger, files, minWords);
|
|
949
|
+
if (flags.json) console.log(JSON.stringify({ ok: issues.length === 0, minWords, checked: files.map((file) => file.label), issues, warnings }, null, 2));
|
|
950
|
+
else {
|
|
951
|
+
console.log(`${issues.length === 0 ? "PASS" : "FAIL"} overlap (${files.length} file(s), runs of ${minWords}+ words)`);
|
|
952
|
+
for (const issue of issues) console.log(` - ${issue}`);
|
|
953
|
+
for (const warning of warnings) console.log(` ~ ${warning}`);
|
|
954
|
+
}
|
|
955
|
+
return issues.length === 0 ? 0 : 1;
|
|
956
|
+
}
|
|
957
|
+
if (command === "brief") {
|
|
958
|
+
try {
|
|
959
|
+
console.log(buildImplementerBrief(run));
|
|
960
|
+
return 0;
|
|
961
|
+
} catch (error) {
|
|
962
|
+
console.error(error.message);
|
|
963
|
+
return 2;
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
if (command === "check") {
|
|
967
|
+
const result = checkRun(run, { phase: flags.phase ?? "report" });
|
|
968
|
+
if (flags.json) console.log(JSON.stringify(result, null, 2));
|
|
969
|
+
else {
|
|
970
|
+
console.log(`${result.ok ? "PASS" : "FAIL"} check through ${result.phase}`);
|
|
971
|
+
for (const issue of result.issues) console.log(` - ${issue}`);
|
|
972
|
+
for (const warning of result.warnings) console.log(` ~ ${warning}`);
|
|
973
|
+
}
|
|
974
|
+
return result.ok ? 0 : 1;
|
|
975
|
+
}
|
|
976
|
+
console.error(USAGE);
|
|
977
|
+
return 64;
|
|
978
|
+
};
|
|
979
|
+
|
|
980
|
+
const invokedDirectly = (() => {
|
|
981
|
+
try {
|
|
982
|
+
return Boolean(process.argv[1]) && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
|
|
983
|
+
} catch {
|
|
984
|
+
return false;
|
|
985
|
+
}
|
|
986
|
+
})();
|
|
987
|
+
|
|
988
|
+
if (invokedDirectly) {
|
|
989
|
+
main(process.argv.slice(2)).then((code) => { process.exitCode = code; }, (error) => {
|
|
990
|
+
console.error(error.message);
|
|
991
|
+
process.exitCode = 1;
|
|
992
|
+
});
|
|
993
|
+
}
|