@jokerized/decksmith 0.3.1 → 0.3.2

package/dist/types/source/harvest.d.ts

@@ -0,0 +1,250 @@
+import { type Source } from "../types.js";
+export interface HarvestOptions {
+    /** Cap on the page's own HTML. Generous: a real article ships megabytes of it. */
+    maxBytes?: number;
+    /** Cap on one asset, matching what `fetchFigures` allows a figure to be. */
+    maxAssetBytes?: number;
+    /** Whole-call budget for each fetch, and for the browser's own steps. */
+    timeoutMs?: number;
+    /** How many assets are downloaded before the rest are named in `warnings`. */
+    maxAssets?: number;
+    /**
+     * How many VIDEOS are downloaded, counted separately from `maxAssets`.
+     *
+     * Separate because the two are different sizes of mistake. A page with forty
+     * images costs a few megabytes; a page with forty videos costs a gigabyte and
+     * an hour, and one of them was probably an advertisement. A clip earns its
+     * place by being the thing a beat is planned around, and a deck does not have
+     * room for four of those, let alone forty.
+     */
+    maxClips?: number;
+    /**
+     * Seconds of a downloaded clip that are kept. Longer is TRUNCATED, and the
+     * trim is warned about rather than performed quietly.
+     *
+     * A cap on SECONDS rather than on bytes because seconds are what the render
+     * spends: hyperframes pre-decodes a clip to one still per output frame before
+     * capture begins, so a five-minute video inside a four-minute deck is four
+     * minutes of full-size stills written to disk for a beat that can be sixty
+     * seconds at most (`beatSchema` in src/types.ts caps it there). The default
+     * lives in ./transcode.ts beside the rest of the encode.
+     *
+     * Ignored when `transcode` is false: trimming is something ffmpeg does, and
+     * there is no ffmpeg in that path to do it.
+     */
+    maxClipSeconds?: number;
+    /**
+     * Whether a downloaded clip is re-encoded to a slide-sized VP9 webm at all.
+     * True unless stated, and stating `false` ships the page's own file.
+     *
+     * The reason to turn it off is that the encode is the one part of a harvest
+     * that costs CPU rather than network: a caller re-ingesting the same page ten
+     * times while tuning a plan pays for it ten times, and the deck it is looking
+     * at does not care. The reason to leave it on is everything ./transcode.ts
+     * says — the bytes shipped, and the stills the render writes.
+     */
+    transcode?: boolean;
+    /**
+     * Total bytes across every asset this harvest downloads.
+     *
+     * `maxAssetBytes` bounds ONE file; nothing bounded the sum, so forty assets
+     * one byte under the per-file cap was a legal harvest of 1.2 GB. Charged from
+     * what actually arrived, and only for a fetch that succeeded — the counter
+     * that charges before the check is how a refused figure still costs the
+     * budget it was refused for (`guardFigures` in src/server/pipeline.ts does
+     * exactly that, deliberately not copied here).
+     */
+    maxTotalBytes?: number;
+    /**
+     * Wall clock for the whole harvest, checked BEFORE each fetch is started.
+     *
+     * BE HONEST ABOUT WHAT THIS BOUNDS: it stops the NEXT fetch, never one already
+     * in flight, so the real ceiling is this plus one `timeoutMs`. Bounding it
+     * exactly would mean an abort signal threaded through `fetchGuarded`, and a
+     * harvest that overruns by twenty seconds is not the failure this is for — a
+     * page whose forty images each take fifteen seconds is.
+     */
+    maxWallMs?: number;
+    /**
+     * How the markdown SPELLS its asset references. Absolute paths by default,
+     * which is what a caller reading the document in place needs.
+     *
+     * `"relative"` writes the bare filename instead, for a caller that is about to
+     * move the directory somewhere this process cannot see — the MCP zips the
+     * harvest and hands it to the server, where an absolute path out of this
+     * machine's temp directory is refused by `guardFigures` and the figure is
+     * dropped. `assets` stays absolute either way: it names files on THIS disk.
+     */
+    refs?: "absolute" | "relative";
+    /**
+     * TEST SEAM, passed straight through to `fetchGuarded`, where it is documented.
+     *
+     * It is the only way to point this at a `node:http` server on loopback, which
+     * is what test/harvest.test.ts needs to drive the real code path — including
+     * the interception proof, which requires a second server this process can see
+     * the request count of. It relaxes nothing else: the private ranges, the
+     * schemes, the caps and the timeout all still apply. Nothing in src passes it.
+     */
+    allowLoopback?: boolean;
+}
+export interface Harvested {
+    /** The document, in the dialect src/source/markdown.ts reads. */
+    markdown: string;
+    /**
+     * Absolute paths of every file the MARKDOWN references, in document order. A
+     * clip's video is written into `dir` too and is deliberately not one of them:
+     * the markdown cannot reference it, and a caller shipping the document
+     * elsewhere (the MCP zips this list) would otherwise carry megabytes nothing
+     * in the document points at. It is named in `clips` instead.
+     */
+    assets: string[];
+    /**
+     * The videos, which the markdown CANNOT carry — see `HarvestedClip`. Hand
+     * these to `attachClips` with the parsed source to get them back.
+     */
+    clips: HarvestedClip[];
+    /** Everything left out, and why. One dead image is not a failed harvest. */
+    warnings: string[];
+    /** The page's title, also emitted as the document's opening `#` heading. */
+    title: string;
+}
+/**
+ * One video, carried BESIDE the markdown because the dialect has no word for it.
+ *
+ * `parseMarkdown` produces figures out of images and nothing else, so a clip —
+ * `kind: "clip"`, a poster, a duration, and either a file or a page to watch it
+ * on — cannot be spelled in the document at all. Rather than invent a dialect
+ * extension that only this module writes and only `parseMarkdown` would have to
+ * learn, the clip travels alongside and `attachClips` puts it back afterwards.
+ *
+ * `poster` is also the JOIN: when there is one, the markdown references it as an
+ * ordinary image, so `parseMarkdown` gives that figure the id, the section and
+ * the sentence that mentions it — everything the planner uses to decide where a
+ * picture belongs — and `attachClips` upgrades that same figure in place. A clip
+ * with no poster has nothing to join to and is appended as a new figure, which
+ * costs it exactly those three facts.
+ */
+export interface HarvestedClip {
+    /** Absolute path of the downloaded video, or "" for one we hold no file for. */
+    file: string;
+    /** Absolute path of the still, or "" when the page offered none. */
+    poster: string;
+    /** Where a viewer watches it, when there is no file. "" when there is one. */
+    href: string;
+    /**
+     * The VIDEO's own pixels when we hold the file, and the poster's when we do
+     * not. types.ts says this box is the video's rather than the still's, and it
+     * is right — every annotation downstream is a fraction of it. A clip we could
+     * not download has no other box to offer, and the deck shows the still.
+     */
+    width: number;
+    height: number;
+    /** Measured off the container, never guessed. Absent for a link-only clip. */
+    seconds?: number;
+    /** The figcaption, link text or iframe title the page gave it. */
+    caption: string;
+}
+/**
+ * One block of the page, as the DOM walker sees it and as `toMarkdown` writes it.
+ *
+ * `src`, `poster` and `href` hold the PAGE's URLs when `readDom` returns them and
+ * LOCAL absolute paths once `localise` has rewritten them. One type rather than
+ * two because the two differ in nothing but that, and a second near-identical
+ * union is the kind of thing that grows a third.
+ */
+export type Block = {
+    kind: "heading";
+    depth: number;
+    text: string;
+} | {
+    kind: "paragraph";
+    text: string;
+} | {
+    kind: "code";
+    text: string;
+} | {
+    kind: "list";
+    ordered: boolean;
+    items: string[];
+} | {
+    kind: "table";
+    columns: string[];
+    rows: string[][];
+} | {
+    kind: "image";
+    src: string;
+    alt: string;
+    caption: string;
+} | {
+    kind: "video";
+    src: string;
+    poster: string;
+    href: string;
+    caption: string;
+};
+export declare function harvest(url: string, dir: string, opts?: HarvestOptions): Promise<Harvested>;
+/** What a container declares about the picture inside it. */
+export interface Measured {
+    width: number;
+    height: number;
+    /** Absent when the container declares no usable duration — a live capture does. */
+    seconds?: number;
+}
+/**
+ * A video's display box and its length, read out of its own header.
+ *
+ * WHY NOT `ffprobe`, WHICH WOULD BE FOUR LINES. Because it would make ingest —
+ * the one verb that has to work on a laptop with a browser and nothing else —
+ * depend on a binary this project otherwise needs only to RENDER. A machine
+ * without ffmpeg would then harvest a page and silently come back with the video
+ * demoted to a link, which is the shape of failure this file exists to avoid. A
+ * width, a height and a duration are four integers in a header; reading them is
+ * cheaper than the dependency, and it is the same trade `imageSize` already
+ * makes for PNG, JPEG, WebP and AVIF.
+ *
+ * Exported because it is pure, and because the half of test/harvest.test.ts that
+ * runs on CI has no browser — measuring a hand-built header is testable there
+ * and driving a real page is not.
+ */
+export declare function videoSize(b: Buffer): Measured & {
+    container: "mp4" | "webm";
+};
+/**
+ * Put the harvest's clips back into the parsed source, and their files beside it.
+ *
+ * CALL THIS BEFORE `fetchFigures`, not after. `fetchFigures` passes a clip
+ * through untouched — it says so in as many words, because a clip carries the
+ * video's dimensions rather than an image's and its first bytes are an `ftyp`
+ * box the image sniffer is right to refuse — so a clip that arrives after it has
+ * run is a figure nothing ever localises, whose `src` is an absolute path into a
+ * temp directory that will not exist on the machine that opens the deck.
+ *
+ * WHERE A CLIP LANDS. One with a poster REPLACES the figure `parseMarkdown` made
+ * out of that poster, keeping its id, its section and the sentence that mentions
+ * it — the three facts the planner uses to decide which point a picture belongs
+ * to, and the reason the poster is written into the markdown at all. One without
+ * a poster has nothing to replace and is appended, which costs it exactly those
+ * three facts and is why a page that gives its videos posters harvests better.
+ */
+export declare function attachClips(source: Source, clips: readonly HarvestedClip[], dir: string): Promise<Source>;
+/**
+ * Blocks become THIS PROJECT'S markdown dialect, which is narrower than markdown.
+ *
+ * Three rules from src/source/markdown.ts, each of which loses content silently
+ * when broken — no error, no gate, just a text-only deck:
+ *
+ *  - A figure is lifted only from a paragraph whose children are ALL images
+ *    (`onlyImages`, around :82 and :158). So an image is always alone in its
+ *    paragraph; `readDom` has already broken sentences around inline ones.
+ *  - A caption is read only from a FOLLOWING paragraph that is a single run of
+ *    emphasis (`captionOf`, :166). So a caption is `*text*` on its own, directly
+ *    after the image, with every `*` inside it escaped.
+ *  - Raw HTML returns the empty string in BOTH walkers (:243 and :264). So
+ *    nothing here emits raw HTML, ever — not a `<figure>`, not a `<br>`, not an
+ *    HTML comment.
+ *
+ * Exported because test/harvest.test.ts runs `parseMarkdown` back over its output
+ * and asserts the figures survive, and that test must run on a machine with no
+ * browser — which is every CI runner this project has.
+ */
+export declare function toMarkdown(blocks: readonly Block[]): string;

package/dist/types/source/readability.d.ts

@@ -0,0 +1,120 @@
+/**
+ * WHICH PART OF A PAGE IS THE ARTICLE — the Readability scoring algorithm,
+ * implemented here rather than depended on.
+ *
+ * `src/source/harvest.ts` picks its content region with three rules: believe
+ * `<main>`, believe `<article>`, else take the deepest element whose block text
+ * minus its link text is maximal. Its own comment says that is deliberately not
+ * a Readability implementation, and it holds up on the shapes it was written
+ * against. It loses on the shapes it was not: a `<main>` that also wraps the
+ * comment thread is believed whole and short-circuits before the loop ever runs;
+ * a `<div role="dialog">` cookie banner matches nothing in the walker's SKIP set,
+ * so it is neither scored down nor skipped; and the score counts text inside
+ * `nav`/`aside`/`footer` subtrees that the walker will then throw away, so a
+ * wrapper can be chosen on the strength of text that never reaches the markdown.
+ *
+ * This is Mozilla Readability's `grabArticle`, in the shape that fits here:
+ * strip the elements whose tag, role, class or id say chrome; score every
+ * paragraph by its length and its comma count; propagate that score to the
+ * parent, the grandparent and up to five levels of ancestor, dividing as it
+ * climbs; discount each candidate by its LINK DENSITY, which is the term that
+ * tells a rail of headlines from prose; take the highest, climb to a parent that
+ * scores higher still, then merge in the siblings that score close to it.
+ *
+ * WHAT THIS IS NOT. It is not a semantic understanding of the page. Nothing here
+ * reads the article; it is arithmetic over text length, punctuation, and the
+ * words site authors happen to put in class names. It is much better than one
+ * subtraction and it is wrong in ways that are structural rather than accidental,
+ * so they are worth naming:
+ *
+ *  - A PAGINATED ARTICLE harvests as page one and nothing says so. Readability
+ *    proper follows "next page" links; this cannot, because `harvest` fetches
+ *    exactly one document and every subresource is aborted.
+ *  - A PAGE WHOSE CONTENT IS A LIST OF LINKS — a link blog, a search result, a
+ *    documentation index — is scored down by the very term that makes the pass
+ *    work. Link density cannot tell a nav from a page that is a nav on purpose.
+ *  - A GALLERY loses. Pictures carry no text, `<li>` is not a scored tag, and a
+ *    page of captioned images scores near zero everywhere, so the winner is
+ *    whichever caption block happened to be longest.
+ *  - CLASS NAMES ARE A GUESS, and the same word list that rescues
+ *    `<div class="article-body">` rescues `<div class="comment-body">`.
+ *  - AN ELEMENT THE STRIP REMOVES TAKES ITS `<video>` WITH IT. That matters more
+ *    here than in a reader: harvest's poster is the join that lets `attachClips`
+ *    keep a clip's id, section and mention, so a silently dropped player demotes
+ *    a clip with every gate green. Hence `mediaDropped` in the report — this pass
+ *    counts what it lost rather than leaving the caller to find out later.
+ *
+ * When it declines it says so, PUTS BACK EVERY NODE IT REMOVED, and changes
+ * nothing else, so `harvest`'s own heuristic runs against the document it would
+ * have seen. Declining is the retry Readability performs with its flags off;
+ * here the simpler pass IS that retry, and it already exists.
+ *
+ * IT RUNS INSIDE THE PAGE. `harvest` ships code into the browser as a function
+ * reference — `page.evaluate(readDom)` — which puppeteer serialises with
+ * `Function.prototype.toString()` and evaluates as source text. `readContentRegion`
+ * is shipped the same way and carries the same rule: it may not close over
+ * anything in this module, so every constant and every helper it uses is declared
+ * inside it, INCLUDING the marker attribute that `CONTENT_MARKER` below spells a
+ * second time. A closed-over constant does not fail here, where a test would see
+ * it; it fails in the page, as a `ReferenceError`, on somebody's ingest — so
+ * test/readability.test.ts asserts the two spellings agree and that the source
+ * names nothing from Node. The node bundle is built with `minify: false`
+ * (scripts/build.mjs), which is what keeps the serialised text intact.
+ */
+/**
+ * The attribute the pass puts on the region it chose.
+ *
+ * A marker rather than a return value because an `Element` cannot cross the
+ * `page.evaluate` boundary — what comes back is structured-clone data. Marking
+ * is also what lets this stay a second self-contained function beside `readDom`
+ * rather than something `readDom` has to grow a second job for: the walker's
+ * `pickRoot` need only look for this attribute first and keep its own three
+ * rules as the fallback for when this pass declines.
+ */
+export declare const CONTENT_MARKER = "data-ds-content";
+/** One element the scorer considered, in the terms it was judged on. */
+export interface Candidate {
+    /** Up to three levels, `div#page > div#content > article.post`, to name it. */
+    path: string;
+    /** Paragraph scores propagated up, plus this element's tag and class weight. */
+    content: number;
+    /** Anchor text over all text, 0..1 — a rail of headlines is near 1. */
+    linkDensity: number;
+    /** `content * (1 - linkDensity)`. This is what the winner is chosen on. */
+    score: number;
+    /** Characters of text held, whitespace collapsed. */
+    text: number;
+}
+/** What the pass hands back across the `page.evaluate` boundary. */
+export interface ContentPick {
+    /** True when some element in the document now carries `CONTENT_MARKER`. */
+    marked: boolean;
+    /**
+     * Why, in a sentence a `warnings` entry can carry verbatim. Populated on
+     * every path, including the ones that change nothing.
+     */
+    reason: string;
+    /** The best few candidates, best first, so a failure names which part broke. */
+    candidates: Candidate[];
+    /** Siblings merged in beside the winner, the winner itself not counted. */
+    merged: number;
+    /** Elements the strip removed. Zero when the pass declined and put them back. */
+    stripped: number;
+    /** Characters of text in the marked region. */
+    text: number;
+    /**
+     * `<video>` and `<iframe>` elements the page had that the region does not.
+     * Not an error — a related-videos rail SHOULD be lost — but the clip path is
+     * load-bearing enough downstream that the number is reported rather than
+     * discovered.
+     */
+    mediaDropped: number;
+}
+/**
+ * Choose the content region, mark it, and report what that cost.
+ *
+ * Evaluated INSIDE THE PAGE, so nothing outside this function body exists at
+ * run time. Type annotations are erased by the build and are the only thing
+ * here that refers to anything above.
+ */
+export declare function readContentRegion(): ContentPick;

package/dist/types/source/transcode.d.ts

@@ -0,0 +1,83 @@
+export interface TranscodeOptions {
+    /** Longest edge of the box to fit inside, in pixels. Default 1280. */
+    maxEdgePx?: number;
+    /** Seconds to keep. A longer video is TRUNCATED, and the trim is warned. */
+    maxSeconds?: number;
+    /** Wall clock for the ffmpeg run before it is killed. Default 10 minutes. */
+    timeoutMs?: number;
+    /**
+     * TEST SEAM: the binary to run. Only ever "ffmpeg" in src.
+     *
+     * It exists so test/transcode.test.ts can prove the skip path by naming a
+     * binary that is not installed, which is the one branch here that cannot be
+     * exercised on a machine where the feature works.
+     */
+    ffmpeg?: string;
+}
+export interface Transcoded {
+    /** The `out` given, or the `input` given back when nothing was done to it. */
+    path: string;
+    /** Measured off the file `path` names — never the arithmetic that asked for it. */
+    width: number;
+    height: number;
+    /** Absent when the container declares no usable duration, as `Measured` says. */
+    seconds?: number;
+    /** False when `path` is still the original's bytes. */
+    transcoded: boolean;
+    /** What was skipped or traded, and why. Empty when nothing was. */
+    warnings: string[];
+}
+/**
+ * The even box `width`x`height` fits inside, never larger than it started.
+ *
+ * FLOOR to even rather than round, on both edges. Even because VP9 in yuv420p
+ * refuses odd dimensions and says so in a way nobody reads as "your width is
+ * odd"; floor because rounding UP a 1919-wide source that needs no scaling at
+ * all would enlarge it by a pixel, and "never upscale" is easier to keep than to
+ * qualify.
+ *
+ * Rounding each edge independently moves the aspect ratio by up to a pixel's
+ * worth, and there is one place downstream where that is visible: claim-figure
+ * picks a full-width layout at exactly `width / height >= 3` (claim-figure.ts:197),
+ * so a source sitting on that boundary can land either side of it afterwards.
+ * That is a layout choice moving one step, not a deck disagreeing with its file
+ * — the figure carries the measured box, so the planner and the emitter still
+ * describe the same rectangle.
+ */
+export declare function fitBox(width: number, height: number, maxEdgePx: number): {
+    width: number;
+    height: number;
+};
+/**
+ * The command, built where it can be read and tested without running anything.
+ *
+ * The house pattern from src/render/ffmpeg.ts: an exported pure args builder and
+ * a one-line `runTool` at the use site, so the flags are assertable on a machine
+ * with no ffmpeg on it.
+ *
+ * `-an` drops the audio track outright. A clip is emitted muted on purpose — the
+ * deck already spends its single audio track on narration (claim-figure.ts:121)
+ * — so carrying the audio is pure bytes for something nothing will ever unmute.
+ *
+ * `-crf 32 -b:v 0` is libvpx-vp9's constant-quality mode; `-deadline good
+ * -cpu-used 4 -row-mt 1` is the speed setting that makes this affordable at
+ * ingest time (4 seconds of 1080p in 0.8s of wall clock, measured) rather than
+ * the several-minutes-per-clip the encoder's defaults cost. The three bitexact
+ * and metadata flags are the determinism story in the header.
+ */
+export declare function transcodeArgs(input: string, out: string, plan: {
+    width: number;
+    height: number;
+    seconds?: number;
+}): string[];
+/**
+ * Shrink `input` into `out`, and say what the result actually is.
+ *
+ * Throws for exactly one thing: an `input` this cannot measure. That is a bug at
+ * the call site rather than a machine missing a tool — `grabVideo` in harvest
+ * measures the same bytes with the same function before it writes them, and
+ * refuses what it cannot read — and there is no honest box to hand back for a
+ * file whose header says nothing. Everything else that can go wrong comes back
+ * as the original plus a warning.
+ */
+export declare function transcode(input: string, out: string, opts?: TranscodeOptions): Promise<Transcoded>;

package/dist/types/types.d.ts

@@ -23,12 +23,19 @@ export declare const refSchema: z.ZodObject<{
 }, z.core.$strip>;
 export declare const figureSchema: z.ZodObject<{
     id: z.ZodString;
+    kind: z.ZodDefault<z.ZodEnum<{
+        image: "image";
+        clip: "clip";
+    }>>;
     src: z.ZodString;
     caption: z.ZodString;
     width: z.ZodInt;
     height: z.ZodInt;
     sectionId: z.ZodOptional<z.ZodString>;
     mention: z.ZodOptional<z.ZodString>;
+    poster: z.ZodOptional<z.ZodString>;
+    seconds: z.ZodOptional<z.ZodNumber>;
+    href: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
 export declare const equationSchema: z.ZodObject<{
     id: z.ZodString;
@@ -59,12 +66,19 @@ export declare const sourceSchema: z.ZodObject<{
     }, z.core.$strip>>;
     figures: z.ZodArray<z.ZodObject<{
         id: z.ZodString;
+        kind: z.ZodDefault<z.ZodEnum<{
+            image: "image";
+            clip: "clip";
+        }>>;
         src: z.ZodString;
         caption: z.ZodString;
         width: z.ZodInt;
         height: z.ZodInt;
         sectionId: z.ZodOptional<z.ZodString>;
         mention: z.ZodOptional<z.ZodString>;
+        poster: z.ZodOptional<z.ZodString>;
+        seconds: z.ZodOptional<z.ZodNumber>;
+        href: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>>;
     equations: z.ZodArray<z.ZodObject<{
         id: z.ZodString;
@@ -1720,12 +1734,19 @@ export declare const packSchema: z.ZodObject<{
         }, z.core.$strip>>;
         figures: z.ZodArray<z.ZodObject<{
             id: z.ZodString;
+            kind: z.ZodDefault<z.ZodEnum<{
+                image: "image";
+                clip: "clip";
+            }>>;
             src: z.ZodString;
             caption: z.ZodString;
             width: z.ZodInt;
             height: z.ZodInt;
             sectionId: z.ZodOptional<z.ZodString>;
             mention: z.ZodOptional<z.ZodString>;
+            poster: z.ZodOptional<z.ZodString>;
+            seconds: z.ZodOptional<z.ZodNumber>;
+            href: z.ZodOptional<z.ZodString>;
         }, z.core.$strip>>;
         equations: z.ZodArray<z.ZodObject<{
             id: z.ZodString;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jokerized/decksmith",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "Turn a source document into an animated explanation deck.",
   "type": "module",
   "license": "MIT",