@jokerized/decksmith 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +208 -15
- package/dist/cli.js +3125 -742
- package/dist/deck-player-element.js +1 -0
- package/dist/deck-player.js +1 -0
- package/dist/deck-runtime.js +27 -2
- package/dist/embed.html +243 -0
- package/dist/index.js +3211 -886
- package/dist/mcp.js +3189 -844
- package/dist/types/deck/player-element.d.ts +1 -0
- package/dist/types/deck/player.d.ts +37 -0
- package/dist/types/deck/protocol.d.ts +80 -0
- package/dist/types/deck/runtime.d.ts +163 -0
- package/dist/types/emit/archetypes/claim-figure.d.ts +0 -7
- package/dist/types/emit/camera.d.ts +14 -6
- package/dist/types/emit/composition.d.ts +13 -0
- package/dist/types/emit/kit.d.ts +66 -8
- package/dist/types/emit/svg.d.ts +27 -0
- package/dist/types/images/providers.d.ts +14 -5
- package/dist/types/index.d.ts +32 -0
- package/dist/types/mcp/tools.d.ts +20 -0
- package/dist/types/net/fetch.d.ts +81 -0
- package/dist/types/pack/media.d.ts +12 -0
- package/dist/types/plan/prompt.d.ts +2 -1
- package/dist/types/server/pipeline.d.ts +51 -8
- package/dist/types/server/upload.d.ts +29 -5
- package/dist/types/source/assets.d.ts +77 -4
- package/dist/types/source/harvest.d.ts +250 -0
- package/dist/types/source/readability.d.ts +120 -0
- package/dist/types/source/transcode.d.ts +83 -0
- package/dist/types/tmpdir.d.ts +33 -0
- package/dist/types/types.d.ts +49 -0
- package/package.json +2 -2
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one guarded door out of this process for a URL somebody else chose.
|
|
3
|
+
*
|
|
4
|
+
* Two holes made this file. `src/source/assets.ts` fetched a figure with a bare
|
|
5
|
+
* `fetch(src)` — no timeout, no size cap, no idea what came back. And
|
|
6
|
+
* `src/server/pipeline.ts` grew an SSRF pre-check, `reachable()`, which resolves
|
|
7
|
+
* the hostname, refuses the private ranges, and then hands the URL to `fetch`,
|
|
8
|
+
* which resolves it again. A pre-check the socket never honours stops the naive
|
|
9
|
+
* class and nothing else: a public host that 302s to 169.254.169.254 walks
|
|
10
|
+
* straight past it, because the address that was checked is not the address that
|
|
11
|
+
* got connected to.
|
|
12
|
+
*
|
|
13
|
+
* WHY THIS IS node:http/node:https AND NOT `fetch`. Exactly one reason, and it
|
|
14
|
+
* is the reason the file exists: `fetch` gives no way to say "connect to THIS
|
|
15
|
+
* address". `http.request` takes a `lookup`, so the address this module
|
|
16
|
+
* validated is the address the socket opens, and the second DNS answer that
|
|
17
|
+
* would have been a rebinding attack is never asked for. Everything else here —
|
|
18
|
+
* the manual redirect walk, the streaming cap, the timeout, the content-type
|
|
19
|
+
* check — could have been written over `fetch`. The pin could not, and
|
|
20
|
+
* pipeline.ts's own comment named that gap and called it unclosable. It was
|
|
21
|
+
* closable; it just needed a different client.
|
|
22
|
+
*
|
|
23
|
+
* WHAT IS STILL OPEN, because a guard that oversells itself is worse than none:
|
|
24
|
+
* - Resolution is `dns.lookup`, the OS resolver, so `/etc/hosts` and any local
|
|
25
|
+
* override are honoured. That is deliberate — it is how this machine resolves
|
|
26
|
+
* every other name — but it does mean a poisoned hosts file is trusted.
|
|
27
|
+
* - A hostname with several A records is refused if ANY of them is blocked, and
|
|
28
|
+
* the socket is pinned to the first. That is stricter than a browser and is
|
|
29
|
+
* the intended trade: a round-robin where one member is internal is not a
|
|
30
|
+
* host this process wants to talk to at all.
|
|
31
|
+
* - Nothing here bounds how many of these run at once. The caller owns that.
|
|
32
|
+
*/
|
|
33
|
+
export declare function isBlockedAddress(ip: string): string | null;
|
|
34
|
+
export interface GuardOptions {
|
|
35
|
+
/**
|
|
36
|
+
* Cap on the body, applied to DECODED bytes — see `readCapped` and `decode`
|
|
37
|
+
* for the two places it is enforced and why it takes both.
|
|
38
|
+
*/
|
|
39
|
+
maxBytes: number;
|
|
40
|
+
/** Whole-call budget: DNS, connect, every hop, and reading the body. */
|
|
41
|
+
timeoutMs: number;
|
|
42
|
+
/**
|
|
43
|
+
* Refuse unless the response's bare media type matches. Matched against the
|
|
44
|
+
* type WITHOUT its parameters, so `/^image\//` and `/^text\/html$/` both do
|
|
45
|
+
* what they look like against `text/html; charset=utf-8`.
|
|
46
|
+
*/
|
|
47
|
+
accept?: RegExp;
|
|
48
|
+
/**
|
|
49
|
+
* Extra request headers, lowercased on the way in. `authorization` and
|
|
50
|
+
* `cookie` are dropped the moment a redirect changes origin.
|
|
51
|
+
*/
|
|
52
|
+
headers?: Record<string, string>;
|
|
53
|
+
/**
|
|
54
|
+
* TEST SEAM, and the only way to point this at a server on this machine.
|
|
55
|
+
*
|
|
56
|
+
* It exempts loopback from the address block and lifts the 80/443 rule, which
|
|
57
|
+
* together is exactly what it takes to drive a `node:http` server on an
|
|
58
|
+
* ephemeral port through this module's real code path — see test/net.test.ts,
|
|
59
|
+
* which is the only caller. It relaxes NOTHING else: link-local, the private
|
|
60
|
+
* ranges, the non-http schemes, the hop cap, the size cap, the timeout and the
|
|
61
|
+
* content-type check all still apply, which is what makes those tests evidence
|
|
62
|
+
* about the guard rather than about a bypass of it. Nothing in src passes it.
|
|
63
|
+
*/
|
|
64
|
+
allowLoopback?: boolean;
|
|
65
|
+
}
|
|
66
|
+
export interface Fetched {
|
|
67
|
+
bytes: Buffer;
|
|
68
|
+
/** The `Content-Type` header as sent, parameters and all — a caller decoding
|
|
69
|
+
* text needs the charset, and dropping it here would lose it for good. */
|
|
70
|
+
contentType: string;
|
|
71
|
+
/** The URL the bytes actually came from, after redirects. What a caller must
|
|
72
|
+
* resolve relative links against, and what it should cache under. */
|
|
73
|
+
url: string;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Fetch `url`, refusing loudly at the first thing that looks wrong.
|
|
77
|
+
*
|
|
78
|
+
* Every refusal names the URL, what was wrong with it, and what to do; a caller
|
|
79
|
+
* surfacing `err.message` to a user is giving them something actionable.
|
|
80
|
+
*/
|
|
81
|
+
export declare function fetchGuarded(url: string, opts: GuardOptions): Promise<Fetched>;
|
|
@@ -37,6 +37,18 @@ export interface MediaPlan {
|
|
|
37
37
|
}
|
|
38
38
|
/** A player URL is one whose host is in the list, or a subdomain of one. */
|
|
39
39
|
export declare function isEmbed(url: string): boolean;
|
|
40
|
+
/**
|
|
41
|
+
* Every origin an embed can come from, deduped.
|
|
42
|
+
*
|
|
43
|
+
* DERIVED FROM THE RULES ABOVE RATHER THAN RESTATED BESIDE THEM, because the
|
|
44
|
+
* served deck's `frame-src` names exactly this list (src/server/http.ts): a host
|
|
45
|
+
* added here and forgotten there is a frame the browser refuses, and CSP refuses
|
|
46
|
+
* it the way CSP refuses everything — an empty rectangle and a line in a console
|
|
47
|
+
* nobody has open.
|
|
48
|
+
*/
|
|
49
|
+
export declare const EMBED_ORIGINS: readonly string[];
|
|
50
|
+
/** The embeddable form of a player page, or undefined when we cannot say. */
|
|
51
|
+
export declare function embedUrl(url: string): string | undefined;
|
|
40
52
|
/**
|
|
41
53
|
* The final policy for one asset.
|
|
42
54
|
*
|
|
@@ -48,6 +48,7 @@ export declare const SYSTEM: string;
|
|
|
48
48
|
*
|
|
49
49
|
* The figure block carries two facts beyond the caption — the section the image
|
|
50
50
|
* sat under and the prose that refers to it — because a planner that cannot see
|
|
51
|
-
* the picture has nothing else to decide what the picture is FOR.
|
|
51
|
+
* the picture has nothing else to decide what the picture is FOR. A clip carries
|
|
52
|
+
* one more: what the deck can actually show of it, which is a still.
|
|
52
53
|
*/
|
|
53
54
|
export declare function renderSource(source: Source): string;
|
|
@@ -1,21 +1,44 @@
|
|
|
1
|
-
import { type ImageProvider, type Runner } from "../index.js";
|
|
1
|
+
import { type HarvestOptions, type ImageProvider, type Runner, type Source } from "../index.js";
|
|
2
2
|
import type { JobOptions } from "./options.js";
|
|
3
3
|
import type { JobHandle, JobResult, Stage } from "./queue.js";
|
|
4
4
|
import { type Upload } from "./upload.js";
|
|
5
5
|
export interface PipelineInput {
|
|
6
|
-
|
|
6
|
+
/**
|
|
7
|
+
* The document that arrived, when a file was posted. Exactly one of this and
|
|
8
|
+
* `url` is set — `parseSubmission` is what guarantees it, and `ingest` refuses
|
|
9
|
+
* loudly rather than guessing if a direct caller hands it both or neither.
|
|
10
|
+
*/
|
|
11
|
+
upload?: Upload;
|
|
12
|
+
/** The page to harvest, when a URL was posted instead of a file. */
|
|
13
|
+
url?: string;
|
|
7
14
|
options: JobOptions;
|
|
8
15
|
/**
|
|
9
16
|
* Whether a figure named by an http(s) URL may be downloaded.
|
|
10
17
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
18
|
+
* ON by default — src/server/main.ts sets `DECKSMITH_FETCH_FIGURES` to true —
|
|
19
|
+
* because a paper's markdown links its images and a deck that silently drops
|
|
20
|
+
* them is not the deck anyone asked for. The flag is therefore NOT the guard;
|
|
21
|
+
* `guardFigures` below is, and it refuses any URL resolving to a private,
|
|
22
|
+
* loopback or link-local address before the fetch is attempted.
|
|
23
|
+
*
|
|
24
|
+
* When it is on, the fetch itself is `fetchGuarded` in src/net/fetch.ts by way
|
|
25
|
+
* of `fetchFigures`: a 32 MB streaming cap and a 20s whole-call timeout at the
|
|
26
|
+
* socket, with the connection PINNED to the address that was validated. That
|
|
27
|
+
* last part is what closed the DNS-rebinding hole an earlier version of this
|
|
28
|
+
* comment called unclosable. `MAX_REMOTE_FIGURES` below is the count.
|
|
29
|
+
*
|
|
30
|
+
* Off, a remote figure is dropped and named in the warnings.
|
|
17
31
|
*/
|
|
18
32
|
fetchRemoteFigures: boolean;
|
|
33
|
+
/**
|
|
34
|
+
* Caps for the harvest of `url`, overriding `HARVEST_LIMITS`.
|
|
35
|
+
*
|
|
36
|
+
* TEST SEAM, the same shape as `imageChain` and `run` below. It is the only
|
|
37
|
+
* way to point a harvest at a `node:http` server on loopback, which is what
|
|
38
|
+
* test/server.test.ts needs to drive this path with no network. Nothing in
|
|
39
|
+
* src/ sets it, so production always gets `HARVEST_LIMITS` exactly.
|
|
40
|
+
*/
|
|
41
|
+
harvest?: HarvestOptions;
|
|
19
42
|
/**
|
|
20
43
|
* The rungs `illustrate` draws through. A test injects the tool's own SVG and
|
|
21
44
|
* nothing else; absent, the stage resolves its providers from the environment
|
|
@@ -33,3 +56,23 @@ export interface PipelineInput {
|
|
|
33
56
|
/** Which rows the step list should have, decided before anything runs. */
|
|
34
57
|
export declare function stagesFor(options: JobOptions): Stage[];
|
|
35
58
|
export declare function runPipeline(job: JobHandle, input: PipelineInput): Promise<JobResult>;
|
|
59
|
+
/**
|
|
60
|
+
* Rewrite every figure's `src` to something safe to read, or drop the figure.
|
|
61
|
+
*
|
|
62
|
+
* `fetchFigures` does `readFile(src)` for anything that is not an http URL, and
|
|
63
|
+
* `src` is whatever the document's markdown said. A document containing
|
|
64
|
+
* `` would otherwise have this process
|
|
65
|
+
* read it — the image sniffer rejects it a moment later, but the read already
|
|
66
|
+
* happened and the error message quotes the path. So relative paths are resolved
|
|
67
|
+
* inside the upload directory and confined there, and everything else is dropped
|
|
68
|
+
* unless remote fetching has been deliberately switched on.
|
|
69
|
+
*
|
|
70
|
+
* Dropped figures leave before the planner sees the source, so no beat can cite
|
|
71
|
+
* one and `assertRefsResolve` has nothing to fail on.
|
|
72
|
+
*
|
|
73
|
+
* EXPORTED FOR ITS TEST, like `safeUrlPath` and `parseRange` in ./http.ts. Its
|
|
74
|
+
* warnings are the only evidence of which rule fired, and they do not reach a
|
|
75
|
+
* `JobHandle` — so reading them through a whole `runPipeline` is impossible and
|
|
76
|
+
* the alternative was to leave the guard untested.
|
|
77
|
+
*/
|
|
78
|
+
export declare function guardFigures(source: Source, root: string, allowRemote: boolean, warnings: string[]): Promise<Source>;
|
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
* What arrives on the socket, treated as hostile.
|
|
3
3
|
*
|
|
4
4
|
* Three jobs, in order: get the bytes off the wire without letting the sender
|
|
5
|
-
* choose how much memory we spend, turn them into a
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* choose how much memory we spend, turn them into a `Submission` — a file or a
|
|
6
|
+
* URL, never both and never neither, plus the option fields — and, if the file
|
|
7
|
+
* is a zip, get its contents onto disk without letting an entry name decide
|
|
8
|
+
* where "onto disk" is.
|
|
8
9
|
*
|
|
9
10
|
* Everything here is pure except `readBody`, which is why the zip half can be
|
|
10
11
|
* tested against an actually malicious archive with no server running.
|
|
@@ -44,7 +45,30 @@ export interface Upload {
|
|
|
44
45
|
fields: Record<string, string>;
|
|
45
46
|
}
|
|
46
47
|
/**
|
|
47
|
-
*
|
|
48
|
+
* What one POST carries: a document, or the address of one.
|
|
49
|
+
*
|
|
50
|
+
* A union rather than two optional fields on one record, because "exactly one of
|
|
51
|
+
* these" IS the rule, and a shape that can hold both — or neither — is a shape
|
|
52
|
+
* every reader downstream has to re-check. `parseSubmission` is the last place
|
|
53
|
+
* that can still answer the client, so both refusals live there and nothing
|
|
54
|
+
* after it asks the question again.
|
|
55
|
+
*
|
|
56
|
+
* `fields` is on both arms because the option fields belong to the REQUEST, and a
|
|
57
|
+
* URL submission has them just as an upload does. On the file arm it is the very
|
|
58
|
+
* same object as `upload.fields`, not a copy — read either; `Upload` keeps its
|
|
59
|
+
* own reference because src/mcp/tools.ts builds one without ever seeing a form.
|
|
60
|
+
*/
|
|
61
|
+
export type Submission = {
|
|
62
|
+
kind: "file";
|
|
63
|
+
upload: Upload;
|
|
64
|
+
fields: Record<string, string>;
|
|
65
|
+
} | {
|
|
66
|
+
kind: "url";
|
|
67
|
+
url: string;
|
|
68
|
+
fields: Record<string, string>;
|
|
69
|
+
};
|
|
70
|
+
/**
|
|
71
|
+
* multipart/form-data, with no dependency, and the exactly-one rule enforced.
|
|
48
72
|
*
|
|
49
73
|
* `new Response(body, { headers }).formData()` is undici's parser, which ships
|
|
50
74
|
* in Node — measured against a hand-built body with a filename containing a
|
|
@@ -52,7 +76,7 @@ export interface Upload {
|
|
|
52
76
|
* with the right bytes. It throws a bare TypeError on a malformed body, which
|
|
53
77
|
* is not a sentence anyone can act on, so it is translated here.
|
|
54
78
|
*/
|
|
55
|
-
export declare function
|
|
79
|
+
export declare function parseSubmission(body: Buffer, contentType: string): Promise<Submission>;
|
|
56
80
|
/** A zip starts "PK". Asked before unzipping so a PDF reads as a PDF. */
|
|
57
81
|
export declare function looksLikeZip(bytes: Uint8Array): boolean;
|
|
58
82
|
/**
|
|
@@ -1,11 +1,84 @@
|
|
|
1
1
|
import { type Source } from "../types.js";
|
|
2
|
-
/** Fetch every figure into `dir`, rewrite `src` to the local name, measure it. */
|
|
3
|
-
export declare function fetchFigures(source: Source, dir: string): Promise<Source>;
|
|
4
2
|
/**
|
|
5
|
-
*
|
|
6
|
-
*
|
|
3
|
+
* Fetch every figure into `dir`, rewrite `src` to the local name, measure it.
|
|
4
|
+
*
|
|
5
|
+
* A figure that cannot be fetched, cannot be measured, or cannot be represented
|
|
6
|
+
* is DROPPED and named in `warnings` — the same shape `guardFigures` in
|
|
7
|
+
* src/server/pipeline.ts already uses for the figures it refuses to fetch at
|
|
8
|
+
* all. The reason is what the caller is: a whole ingest. A paper with eleven
|
|
9
|
+
* figures and one dead CDN link is still the deck someone asked for, and before
|
|
10
|
+
* this the dead link ended the run with `unrecognised image header` and no
|
|
11
|
+
* indication of which figure had produced it.
|
|
12
|
+
*
|
|
13
|
+
* `warnings` is an out-parameter rather than a second return value so the two
|
|
14
|
+
* callers that do not collect them (`decksmith ingest`, and the pipeline's own
|
|
15
|
+
* call) keep compiling unchanged, and so a caller that does collect them can
|
|
16
|
+
* pass the SAME array it already threads through `guardFigures` and have both
|
|
17
|
+
* sets of drops read as one list. Omit it and the drops go to stderr instead —
|
|
18
|
+
* dropping a figure only stops being an improvement over aborting if somebody
|
|
19
|
+
* is told which figure went.
|
|
20
|
+
*/
|
|
21
|
+
export declare function fetchFigures(source: Source, dir: string, warnings?: string[]): Promise<Source>;
|
|
22
|
+
/** Every format this reads. Also every format a page is likely to serve a figure in. */
|
|
23
|
+
export type ImageFormat = "png" | "jpeg" | "gif" | "webp" | "avif" | "svg";
|
|
24
|
+
/**
|
|
25
|
+
* The format the bytes claim to be, from the header alone.
|
|
26
|
+
*
|
|
27
|
+
* Exported and separate from `imageSize` because two callers ask two different
|
|
28
|
+
* questions of the same eight bytes: what to name the file, and how to measure
|
|
29
|
+
* it. Keeping them one function is how a file ends up named after the reader
|
|
30
|
+
* that happened to succeed.
|
|
31
|
+
*/
|
|
32
|
+
export declare function sniffFormat(b: Buffer): ImageFormat | undefined;
|
|
33
|
+
/**
|
|
34
|
+
* Intrinsic size from the file header, for every format a page may hand us.
|
|
35
|
+
*
|
|
36
|
+
* PNG, JPEG and GIF were enough while a source was a markdown file with its own
|
|
37
|
+
* images beside it. A URL is not: a page today serves WebP for the photograph,
|
|
38
|
+
* AVIF for the hero and SVG for the diagram, and a reader that knows three
|
|
39
|
+
* formats measures the other three as "unrecognised" and drops them.
|
|
40
|
+
*
|
|
41
|
+
* Still no dependency for this. Each reader below is a handful of field reads
|
|
42
|
+
* against a published header layout, and the alternative is a decoder that
|
|
43
|
+
* pulls in the whole pixel pipeline to answer two integers.
|
|
44
|
+
*
|
|
45
|
+
* It THROWS rather than returning zeros, and the message names what it saw.
|
|
46
|
+
* `figureSchema` requires a positive int, so a zero would not be caught here at
|
|
47
|
+
* all — it would surface much later as `width: too small` against a figure id,
|
|
48
|
+
* with nothing to say which URL had produced it.
|
|
7
49
|
*/
|
|
8
50
|
export declare function imageSize(b: Buffer): {
|
|
9
51
|
width: number;
|
|
10
52
|
height: number;
|
|
11
53
|
};
|
|
54
|
+
/**
|
|
55
|
+
* The size an SVG declares, in CSS pixels.
|
|
56
|
+
*
|
|
57
|
+
* Two callers: figure ingest above, and `sizeOf` in src/images/illustrate.ts
|
|
58
|
+
* through the re-export in src/images/providers.ts, where this used to live as a
|
|
59
|
+
* regex for `viewBox="0 0 %d %d"` over the first 200 bytes. That was honest for
|
|
60
|
+
* the SVG the tool draws itself and wrong for everything else, because a real
|
|
61
|
+
* SVG opens with a licence comment above the root element, spells the viewBox
|
|
62
|
+
* with floats, writes `width="640px"`, or carries no viewBox at all.
|
|
63
|
+
*
|
|
64
|
+
* `width`/`height` win when BOTH resolve to an absolute length, because that is
|
|
65
|
+
* the intrinsic size a browser would use. The viewBox is the fallback, and it is
|
|
66
|
+
* also the right answer for the very common `width="100%"`: a percentage is a
|
|
67
|
+
* fraction of a viewport this pipeline has not created yet, so it is deliberately
|
|
68
|
+
* left unresolved rather than read as 100 pixels.
|
|
69
|
+
*
|
|
70
|
+
* CHECKED against the browser that draws these: 60 real SVGs off this machine
|
|
71
|
+
* measured here and loaded in Chrome, and all 60 agree with `naturalWidth` and
|
|
72
|
+
* `naturalHeight` to the pixel. Two deliberate divergences remain. A fractional
|
|
73
|
+
* viewBox is rounded, where Chrome's own rounding differs by up to a pixel in
|
|
74
|
+
* either direction (300.5 → 300, 300.4 → 299, 99.9 → 100), which is a ratio
|
|
75
|
+
* unchanged in the third decimal. And an SVG declaring NEITHER a size nor a
|
|
76
|
+
* viewBox is refused rather than given the 300x150 a browser hands a replaced
|
|
77
|
+
* element with no intrinsic size: that default is a CSS fallback, not a fact
|
|
78
|
+
* about the picture, and everything downstream frames the figure against the box
|
|
79
|
+
* recorded here.
|
|
80
|
+
*/
|
|
81
|
+
export declare function svgSize(bytes: Buffer): {
|
|
82
|
+
width: number;
|
|
83
|
+
height: number;
|
|
84
|
+
};
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
import { type Source } from "../types.js";
|
|
2
|
+
export interface HarvestOptions {
|
|
3
|
+
/** Cap on the page's own HTML. Generous: a real article ships megabytes of it. */
|
|
4
|
+
maxBytes?: number;
|
|
5
|
+
/** Cap on one asset, matching what `fetchFigures` allows a figure to be. */
|
|
6
|
+
maxAssetBytes?: number;
|
|
7
|
+
/** Whole-call budget for each fetch, and for the browser's own steps. */
|
|
8
|
+
timeoutMs?: number;
|
|
9
|
+
/** How many assets are downloaded before the rest are named in `warnings`. */
|
|
10
|
+
maxAssets?: number;
|
|
11
|
+
/**
|
|
12
|
+
* How many VIDEOS are downloaded, counted separately from `maxAssets`.
|
|
13
|
+
*
|
|
14
|
+
* Separate because the two are different sizes of mistake. A page with forty
|
|
15
|
+
* images costs a few megabytes; a page with forty videos costs a gigabyte and
|
|
16
|
+
* an hour, and one of them was probably an advertisement. A clip earns its
|
|
17
|
+
* place by being the thing a beat is planned around, and a deck does not have
|
|
18
|
+
* room for four of those, let alone forty.
|
|
19
|
+
*/
|
|
20
|
+
maxClips?: number;
|
|
21
|
+
/**
|
|
22
|
+
* Seconds of a downloaded clip that are kept. Longer is TRUNCATED, and the
|
|
23
|
+
* trim is warned about rather than performed quietly.
|
|
24
|
+
*
|
|
25
|
+
* A cap on SECONDS rather than on bytes because seconds are what the render
|
|
26
|
+
* spends: hyperframes pre-decodes a clip to one still per output frame before
|
|
27
|
+
* capture begins, so a five-minute video inside a four-minute deck is four
|
|
28
|
+
* minutes of full-size stills written to disk for a beat that can be sixty
|
|
29
|
+
* seconds at most (`beatSchema` in src/types.ts caps it there). The default
|
|
30
|
+
* lives in ./transcode.ts beside the rest of the encode.
|
|
31
|
+
*
|
|
32
|
+
* Ignored when `transcode` is false: trimming is something ffmpeg does, and
|
|
33
|
+
* there is no ffmpeg in that path to do it.
|
|
34
|
+
*/
|
|
35
|
+
maxClipSeconds?: number;
|
|
36
|
+
/**
|
|
37
|
+
* Whether a downloaded clip is re-encoded to a slide-sized VP9 webm at all.
|
|
38
|
+
* True unless stated, and stating `false` ships the page's own file.
|
|
39
|
+
*
|
|
40
|
+
* The reason to turn it off is that the encode is the one part of a harvest
|
|
41
|
+
* that costs CPU rather than network: a caller re-ingesting the same page ten
|
|
42
|
+
* times while tuning a plan pays for it ten times, and the deck it is looking
|
|
43
|
+
* at does not care. The reason to leave it on is everything ./transcode.ts
|
|
44
|
+
* says — the bytes shipped, and the stills the render writes.
|
|
45
|
+
*/
|
|
46
|
+
transcode?: boolean;
|
|
47
|
+
/**
|
|
48
|
+
* Total bytes across every asset this harvest downloads.
|
|
49
|
+
*
|
|
50
|
+
* `maxAssetBytes` bounds ONE file; nothing bounded the sum, so forty assets
|
|
51
|
+
* one byte under the per-file cap was a legal harvest of 1.2 GB. Charged from
|
|
52
|
+
* what actually arrived, and only for a fetch that succeeded — the counter
|
|
53
|
+
* that charges before the check is how a refused figure still costs the
|
|
54
|
+
* budget it was refused for (`guardFigures` in src/server/pipeline.ts does
|
|
55
|
+
* exactly that, deliberately not copied here).
|
|
56
|
+
*/
|
|
57
|
+
maxTotalBytes?: number;
|
|
58
|
+
/**
|
|
59
|
+
* Wall clock for the whole harvest, checked BEFORE each fetch is started.
|
|
60
|
+
*
|
|
61
|
+
* BE HONEST ABOUT WHAT THIS BOUNDS: it stops the NEXT fetch, never one already
|
|
62
|
+
* in flight, so the real ceiling is this plus one `timeoutMs`. Bounding it
|
|
63
|
+
* exactly would mean an abort signal threaded through `fetchGuarded`, and a
|
|
64
|
+
* harvest that overruns by twenty seconds is not the failure this is for — a
|
|
65
|
+
* page whose forty images each take fifteen seconds is.
|
|
66
|
+
*/
|
|
67
|
+
maxWallMs?: number;
|
|
68
|
+
/**
|
|
69
|
+
* How the markdown SPELLS its asset references. Absolute paths by default,
|
|
70
|
+
* which is what a caller reading the document in place needs.
|
|
71
|
+
*
|
|
72
|
+
* `"relative"` writes the bare filename instead, for a caller that is about to
|
|
73
|
+
* move the directory somewhere this process cannot see — the MCP zips the
|
|
74
|
+
* harvest and hands it to the server, where an absolute path out of this
|
|
75
|
+
* machine's temp directory is refused by `guardFigures` and the figure is
|
|
76
|
+
* dropped. `assets` stays absolute either way: it names files on THIS disk.
|
|
77
|
+
*/
|
|
78
|
+
refs?: "absolute" | "relative";
|
|
79
|
+
/**
|
|
80
|
+
* TEST SEAM, passed straight through to `fetchGuarded`, where it is documented.
|
|
81
|
+
*
|
|
82
|
+
* It is the only way to point this at a `node:http` server on loopback, which
|
|
83
|
+
* is what test/harvest.test.ts needs to drive the real code path — including
|
|
84
|
+
* the interception proof, which requires a second server this process can see
|
|
85
|
+
* the request count of. It relaxes nothing else: the private ranges, the
|
|
86
|
+
* schemes, the caps and the timeout all still apply. Nothing in src passes it.
|
|
87
|
+
*/
|
|
88
|
+
allowLoopback?: boolean;
|
|
89
|
+
}
|
|
90
|
+
export interface Harvested {
|
|
91
|
+
/** The document, in the dialect src/source/markdown.ts reads. */
|
|
92
|
+
markdown: string;
|
|
93
|
+
/**
|
|
94
|
+
* Absolute paths of every file the MARKDOWN references, in document order. A
|
|
95
|
+
* clip's video is written into `dir` too and is deliberately not one of them:
|
|
96
|
+
* the markdown cannot reference it, and a caller shipping the document
|
|
97
|
+
* elsewhere (the MCP zips this list) would otherwise carry megabytes nothing
|
|
98
|
+
* in the document points at. It is named in `clips` instead.
|
|
99
|
+
*/
|
|
100
|
+
assets: string[];
|
|
101
|
+
/**
|
|
102
|
+
* The videos, which the markdown CANNOT carry — see `HarvestedClip`. Hand
|
|
103
|
+
* these to `attachClips` with the parsed source to get them back.
|
|
104
|
+
*/
|
|
105
|
+
clips: HarvestedClip[];
|
|
106
|
+
/** Everything left out, and why. One dead image is not a failed harvest. */
|
|
107
|
+
warnings: string[];
|
|
108
|
+
/** The page's title, also emitted as the document's opening `#` heading. */
|
|
109
|
+
title: string;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* One video, carried BESIDE the markdown because the dialect has no word for it.
|
|
113
|
+
*
|
|
114
|
+
* `parseMarkdown` produces figures out of images and nothing else, so a clip —
|
|
115
|
+
* `kind: "clip"`, a poster, a duration, and either a file or a page to watch it
|
|
116
|
+
* on — cannot be spelled in the document at all. Rather than invent a dialect
|
|
117
|
+
* extension that only this module writes and only `parseMarkdown` would have to
|
|
118
|
+
* learn, the clip travels alongside and `attachClips` puts it back afterwards.
|
|
119
|
+
*
|
|
120
|
+
* `poster` is also the JOIN: when there is one, the markdown references it as an
|
|
121
|
+
* ordinary image, so `parseMarkdown` gives that figure the id, the section and
|
|
122
|
+
* the sentence that mentions it — everything the planner uses to decide where a
|
|
123
|
+
* picture belongs — and `attachClips` upgrades that same figure in place. A clip
|
|
124
|
+
* with no poster has nothing to join to and is appended as a new figure, which
|
|
125
|
+
* costs it exactly those three facts.
|
|
126
|
+
*/
|
|
127
|
+
export interface HarvestedClip {
|
|
128
|
+
/** Absolute path of the downloaded video, or "" for one we hold no file for. */
|
|
129
|
+
file: string;
|
|
130
|
+
/** Absolute path of the still, or "" when the page offered none. */
|
|
131
|
+
poster: string;
|
|
132
|
+
/** Where a viewer watches it, when there is no file. "" when there is one. */
|
|
133
|
+
href: string;
|
|
134
|
+
/**
|
|
135
|
+
* The VIDEO's own pixels when we hold the file, and the poster's when we do
|
|
136
|
+
* not. types.ts says this box is the video's rather than the still's, and it
|
|
137
|
+
* is right — every annotation downstream is a fraction of it. A clip we could
|
|
138
|
+
* not download has no other box to offer, and the deck shows the still.
|
|
139
|
+
*/
|
|
140
|
+
width: number;
|
|
141
|
+
height: number;
|
|
142
|
+
/** Measured off the container, never guessed. Absent for a link-only clip. */
|
|
143
|
+
seconds?: number;
|
|
144
|
+
/** The figcaption, link text or iframe title the page gave it. */
|
|
145
|
+
caption: string;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* One block of the page, as the DOM walker sees it and as `toMarkdown` writes it.
|
|
149
|
+
*
|
|
150
|
+
* `src`, `poster` and `href` hold the PAGE's URLs when `readDom` returns them and
|
|
151
|
+
* LOCAL absolute paths once `localise` has rewritten them. One type rather than
|
|
152
|
+
* two because the two differ in nothing but that, and a second near-identical
|
|
153
|
+
* union is the kind of thing that grows a third.
|
|
154
|
+
*/
|
|
155
|
+
export type Block = {
|
|
156
|
+
kind: "heading";
|
|
157
|
+
depth: number;
|
|
158
|
+
text: string;
|
|
159
|
+
} | {
|
|
160
|
+
kind: "paragraph";
|
|
161
|
+
text: string;
|
|
162
|
+
} | {
|
|
163
|
+
kind: "code";
|
|
164
|
+
text: string;
|
|
165
|
+
} | {
|
|
166
|
+
kind: "list";
|
|
167
|
+
ordered: boolean;
|
|
168
|
+
items: string[];
|
|
169
|
+
} | {
|
|
170
|
+
kind: "table";
|
|
171
|
+
columns: string[];
|
|
172
|
+
rows: string[][];
|
|
173
|
+
} | {
|
|
174
|
+
kind: "image";
|
|
175
|
+
src: string;
|
|
176
|
+
alt: string;
|
|
177
|
+
caption: string;
|
|
178
|
+
} | {
|
|
179
|
+
kind: "video";
|
|
180
|
+
src: string;
|
|
181
|
+
poster: string;
|
|
182
|
+
href: string;
|
|
183
|
+
caption: string;
|
|
184
|
+
};
|
|
185
|
+
export declare function harvest(url: string, dir: string, opts?: HarvestOptions): Promise<Harvested>;
|
|
186
|
+
/** What a container declares about the picture inside it. */
|
|
187
|
+
export interface Measured {
|
|
188
|
+
width: number;
|
|
189
|
+
height: number;
|
|
190
|
+
/** Absent when the container declares no usable duration — a live capture does. */
|
|
191
|
+
seconds?: number;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* A video's display box and its length, read out of its own header.
|
|
195
|
+
*
|
|
196
|
+
* WHY NOT `ffprobe`, WHICH WOULD BE FOUR LINES. Because it would make ingest —
|
|
197
|
+
* the one verb that has to work on a laptop with a browser and nothing else —
|
|
198
|
+
* depend on a binary this project otherwise needs only to RENDER. A machine
|
|
199
|
+
* without ffmpeg would then harvest a page and silently come back with the video
|
|
200
|
+
* demoted to a link, which is the shape of failure this file exists to avoid. A
|
|
201
|
+
* width, a height and a duration are four integers in a header; reading them is
|
|
202
|
+
* cheaper than the dependency, and it is the same trade `imageSize` already
|
|
203
|
+
* makes for PNG, JPEG, WebP and AVIF.
|
|
204
|
+
*
|
|
205
|
+
* Exported because it is pure, and because the half of test/harvest.test.ts that
|
|
206
|
+
* runs on CI has no browser — measuring a hand-built header is testable there
|
|
207
|
+
* and driving a real page is not.
|
|
208
|
+
*/
|
|
209
|
+
export declare function videoSize(b: Buffer): Measured & {
|
|
210
|
+
container: "mp4" | "webm";
|
|
211
|
+
};
|
|
212
|
+
/**
|
|
213
|
+
* Put the harvest's clips back into the parsed source, and their files beside it.
|
|
214
|
+
*
|
|
215
|
+
* CALL THIS BEFORE `fetchFigures`, not after. `fetchFigures` passes a clip
|
|
216
|
+
* through untouched — it says so in as many words, because a clip carries the
|
|
217
|
+
* video's dimensions rather than an image's and its first bytes are an `ftyp`
|
|
218
|
+
* box the image sniffer is right to refuse — so a clip that arrives after it has
|
|
219
|
+
* run is a figure nothing ever localises, whose `src` is an absolute path into a
|
|
220
|
+
* temp directory that will not exist on the machine that opens the deck.
|
|
221
|
+
*
|
|
222
|
+
* WHERE A CLIP LANDS. One with a poster REPLACES the figure `parseMarkdown` made
|
|
223
|
+
* out of that poster, keeping its id, its section and the sentence that mentions
|
|
224
|
+
* it — the three facts the planner uses to decide which point a picture belongs
|
|
225
|
+
* to, and the reason the poster is written into the markdown at all. One without
|
|
226
|
+
* a poster has nothing to replace and is appended, which costs it exactly those
|
|
227
|
+
* three facts and is why a page that gives its videos posters harvests better.
|
|
228
|
+
*/
|
|
229
|
+
export declare function attachClips(source: Source, clips: readonly HarvestedClip[], dir: string): Promise<Source>;
|
|
230
|
+
/**
|
|
231
|
+
* Blocks become THIS PROJECT'S markdown dialect, which is narrower than markdown.
|
|
232
|
+
*
|
|
233
|
+
* Three rules from src/source/markdown.ts, each of which loses content silently
|
|
234
|
+
* when broken — no error, no gate, just a text-only deck:
|
|
235
|
+
*
|
|
236
|
+
* - A figure is lifted only from a paragraph whose children are ALL images
|
|
237
|
+
* (`onlyImages`, around :82 and :158). So an image is always alone in its
|
|
238
|
+
* paragraph; `readDom` has already broken sentences around inline ones.
|
|
239
|
+
* - A caption is read only from a FOLLOWING paragraph that is a single run of
|
|
240
|
+
* emphasis (`captionOf`, :166). So a caption is `*text*` on its own, directly
|
|
241
|
+
* after the image, with every `*` inside it escaped.
|
|
242
|
+
* - Raw HTML returns the empty string in BOTH walkers (:243 and :264). So
|
|
243
|
+
* nothing here emits raw HTML, ever — not a `<figure>`, not a `<br>`, not an
|
|
244
|
+
* HTML comment.
|
|
245
|
+
*
|
|
246
|
+
* Exported because test/harvest.test.ts runs `parseMarkdown` back over its output
|
|
247
|
+
* and asserts the figures survive, and that test must run on a machine with no
|
|
248
|
+
* browser — which is every CI runner this project has.
|
|
249
|
+
*/
|
|
250
|
+
export declare function toMarkdown(blocks: readonly Block[]): string;
|