@plurnk/plurnk-schemes-http 1.3.12 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/.env.defaults +18 -46
  2. package/README.md +74 -23
  3. package/SPEC.md +487 -124
  4. package/dist/Config.d.ts +3 -0
  5. package/dist/Config.d.ts.map +1 -0
  6. package/dist/Config.js +18 -0
  7. package/dist/Config.js.map +1 -0
  8. package/dist/ContentType.d.ts +2 -0
  9. package/dist/ContentType.d.ts.map +1 -0
  10. package/dist/ContentType.js +15 -0
  11. package/dist/ContentType.js.map +1 -0
  12. package/dist/ErrorDetail.d.ts +6 -0
  13. package/dist/ErrorDetail.d.ts.map +1 -0
  14. package/dist/ErrorDetail.js +19 -0
  15. package/dist/ErrorDetail.js.map +1 -0
  16. package/dist/Guard.d.ts +1 -1
  17. package/dist/Guard.d.ts.map +1 -1
  18. package/dist/Guard.js +95 -55
  19. package/dist/Guard.js.map +1 -1
  20. package/dist/Http.d.ts +3 -16
  21. package/dist/Http.d.ts.map +1 -1
  22. package/dist/Http.js +817 -331
  23. package/dist/Http.js.map +1 -1
  24. package/dist/TavilyExtract.d.ts +35 -0
  25. package/dist/TavilyExtract.d.ts.map +1 -0
  26. package/dist/TavilyExtract.js +204 -0
  27. package/dist/TavilyExtract.js.map +1 -0
  28. package/dist/WebFetcher.d.ts +88 -18
  29. package/dist/WebFetcher.d.ts.map +1 -1
  30. package/dist/WebFetcher.js +588 -70
  31. package/dist/WebFetcher.js.map +1 -1
  32. package/dist/Ws.d.ts +7 -3
  33. package/dist/Ws.d.ts.map +1 -1
  34. package/dist/Ws.js +432 -96
  35. package/dist/Ws.js.map +1 -1
  36. package/dist/index.d.ts +2 -3
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +4 -10
  39. package/dist/index.js.map +1 -1
  40. package/docs/http.md +108 -53
  41. package/docs/wss.md +39 -43
  42. package/package.json +14 -13
  43. package/dist/Browser.d.ts +0 -84
  44. package/dist/Browser.d.ts.map +0 -1
  45. package/dist/Browser.js +0 -273
  46. package/dist/Browser.js.map +0 -1
  47. package/scripts/install-browser.mjs +0 -19
package/dist/Http.js CHANGED
@@ -1,57 +1,151 @@
1
- // http(s):// scheme handler — the first greenfield `@plurnk/plurnk-schemes-*`
2
- // sibling, authored entirely against the stable SchemeCtx contract. It never
3
- // imports private service modules or depends on database layout.
4
- //
5
- // Surface — the HTTP method is the OP (grammar#46): READ→GET, SEND→POST,
6
- // EDIT→PUT, KILL→DELETE. Every request streams its response the same way
7
- // (102 Processing now; the subscription accumulates; the model reads next turn).
8
- // READ(http(s)://host/path) — GET. HTML is rendered; else raw bytes stream.
9
- // SEND[200](http(s)://...) — POST the body; response streams back.
10
- // EDIT(http(s)://...):body: — PUT the body (full-resource replace; no `<L>`).
11
- // KILL(http(s)://...) — DELETE the resource.
12
- // SEND[499](http(s)://...) — cancel an in-flight request (abort the fetch).
13
- // SEND[410](http(s)://...) — delete the locally cached response entry
14
- // (loop disposition, NOT an HTTP DELETE — that's KILL).
15
- //
16
- // Request headers ride IN the target as trailing `{Key: value}` blocks
17
- // (grammar#46 — `UrlPath.headers`, ordered pairs), one header per block:
18
- // READ(https://api.x/v1{Authorization: Bearer T}{Accept: application/json})
19
- // The SEND `[code]` is loop disposition (102/200/…), never the HTTP status —
20
- // the real 2xx/4xx comes back in the response `header`/`body` channels.
21
- //
22
- // `fetch` is the scheme's purpose. Node owns acquisition; eventsource-parser
23
- // owns the WHATWG event-stream framing used by SSE responses.
1
+ // HTTP(S) handler. Operation-to-method semantics live in {§op-surface};
2
+ // acquisition, materialization, publication, and query flow live in
3
+ // {§http-lifecycle}. The implementation depends only on SchemeCtx capabilities.
24
4
  var _a;
25
5
  import { createParser } from "eventsource-parser";
26
- import { Results } from "@plurnk/plurnk-schemes";
6
+ import { MimetypeClassifier, NetworkAddress, ProjectionInputLimitError, Results, } from "@plurnk/plurnk-schemes";
27
7
  import { readFile } from "node:fs/promises";
28
- import Browser, { BROWSER_UA, requireNumEnv } from "./Browser.js";
29
- import WebFetcher from "./WebFetcher.js";
8
+ import ErrorDetail from "./ErrorDetail.js";
9
+ import WebFetcher, { DEFAULT_WEB_UA, CACHE_VARIANT_HEADER, MATERIALIZER_ID_HEADER, PROJECTION_ID_HEADER, TAVILY_CREDITS_HEADER, TAVILY_REQUEST_ID_HEADER, cacheVariantEvidence, classifyCacheVariant, WebMaterializationError, } from "./WebFetcher.js";
10
+ import { responseMimetype } from "./ContentType.js";
11
+ import { requireNonNegativeIntegerEnv as requireNumEnv } from "./Config.js";
30
12
  // The channel the response body streams into, and the header metadata channel.
31
13
  const BODY = "body";
32
14
  const HEADER = "header";
33
- // Materialization stamp line in the HEADER channel — the ONE timestamp the
34
- // freshness predicate reads (SPEC §revalidation). Namespaced so it can never
35
- // collide with a real response header.
15
+ // Package-owned metadata appended after untrusted origin headers. Readers take
16
+ // the last value so an origin using the same field name cannot override it.
36
17
  const FETCHED_AT = "x-plurnk-fetched-at";
37
- // Deep doc lives in `docs/http.md` (the constellation's docs/<name>.md
38
- // convention) and is loaded into the manifest at module init — the contract
39
- // field stays a plain string; only the authoring source moves out of line.
40
- // `../docs/http.md` resolves identically from src/ (test) and dist/ (published):
41
- // both sit one level under the package root. Missing file → fail-hard at import.
18
+ const REQUEST_METHOD = "x-plurnk-request-method";
19
+ const DELTA_SECONDS_LIMIT = 2147483648n;
20
+ const BODY_PROCESSING_FIELDS = new Set([
21
+ "content-encoding",
22
+ "content-length",
23
+ "content-range",
24
+ "content-type",
25
+ ]);
26
+ const splitHttpList = (value) => {
27
+ const members = [];
28
+ let start = 0;
29
+ let quoted = false;
30
+ let escaped = false;
31
+ for (let index = 0; index < value.length; index += 1) {
32
+ const character = value[index];
33
+ if (quoted) {
34
+ if (escaped)
35
+ escaped = false;
36
+ else if (character === "\\")
37
+ escaped = true;
38
+ else if (character === '"')
39
+ quoted = false;
40
+ }
41
+ else if (character === '"') {
42
+ quoted = true;
43
+ }
44
+ else if (character === ",") {
45
+ members.push(value.slice(start, index).trim());
46
+ start = index + 1;
47
+ }
48
+ }
49
+ members.push(value.slice(start).trim());
50
+ return members.filter((member) => member.length > 0);
51
+ };
52
+ const cacheDirective = (member) => {
53
+ const equals = member.indexOf("=");
54
+ const name = member.slice(0, equals < 0 ? undefined : equals).trim().toLowerCase();
55
+ if (!/^[!#$%&'*+.^_`|~0-9a-z-]+$/.test(name))
56
+ return null;
57
+ if (equals < 0)
58
+ return { name };
59
+ return { name, argument: member.slice(equals + 1).trim() };
60
+ };
61
+ const deltaMilliseconds = (argument) => {
62
+ if (argument === undefined)
63
+ return null;
64
+ const match = /^(?:([0-9]+)|"([0-9]+)")$/.exec(argument);
65
+ const digits = match?.[1] ?? match?.[2];
66
+ if (digits === undefined)
67
+ return null;
68
+ const seconds = BigInt(digits);
69
+ return Number(seconds > DELTA_SECONDS_LIMIT ? DELTA_SECONDS_LIMIT : seconds) * 1000;
70
+ };
71
+ const lastHeaderValue = (header, name) => {
72
+ const lines = header.split(/\r?\n/);
73
+ for (let index = lines.length - 1; index >= 0; index -= 1) {
74
+ const line = lines[index];
75
+ const colon = line.indexOf(":");
76
+ if (colon < 0 || line.slice(0, colon).trim().toLowerCase() !== name)
77
+ continue;
78
+ return line.slice(colon + 1).trim();
79
+ }
80
+ return undefined;
81
+ };
82
+ const entityTag = (value) => {
83
+ const match = /^(W\/)?"([\x21\x23-\x7e\x80-\xff]*)"$/.exec(value);
84
+ return match === null
85
+ ? null
86
+ : { weak: match[1] !== undefined, opaque: match[2] };
87
+ };
88
+ const MONTHS = [
89
+ "Jan", "Feb", "Mar", "Apr", "May", "Jun",
90
+ "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
91
+ ];
92
+ const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
93
+ const IMF_FIXDATE = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat), ([0-9]{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) ([0-9]{4}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) GMT$/;
94
+ const RFC850_DATE = /^(Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday), ([0-9]{2})-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) GMT$/;
95
+ const ASCTIME_DATE = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (?: ([0-9])|([0-9]{2})) ([0-9]{2}):([0-9]{2}):([0-9]{2}) ([0-9]{4})$/;
96
+ const utcHttpDate = (weekday, day, month, year, hour, minute, second) => {
97
+ const monthIndex = MONTHS.indexOf(month);
98
+ const weekdayIndex = WEEKDAYS.indexOf(weekday.slice(0, 3));
99
+ const numericDay = Number(day);
100
+ const numericHour = Number(hour);
101
+ const numericMinute = Number(minute);
102
+ const numericSecond = Number(second);
103
+ if (monthIndex < 0 || weekdayIndex < 0)
104
+ return null;
105
+ const date = new Date(0);
106
+ date.setUTCFullYear(year, monthIndex, numericDay);
107
+ date.setUTCHours(numericHour, numericMinute, numericSecond, 0);
108
+ return date.getUTCFullYear() === year
109
+ && date.getUTCMonth() === monthIndex
110
+ && date.getUTCDate() === numericDay
111
+ && date.getUTCHours() === numericHour
112
+ && date.getUTCMinutes() === numericMinute
113
+ && date.getUTCSeconds() === numericSecond
114
+ && date.getUTCDay() === weekdayIndex
115
+ ? date.getTime()
116
+ : null;
117
+ };
118
+ const httpDate = (value, now = Date.now()) => {
119
+ const preferred = IMF_FIXDATE.exec(value);
120
+ if (preferred !== null) {
121
+ return utcHttpDate(preferred[1], preferred[2], preferred[3], Number(preferred[4]), preferred[5], preferred[6], preferred[7]);
122
+ }
123
+ const rfc850 = RFC850_DATE.exec(value);
124
+ if (rfc850 !== null) {
125
+ let year = 2000 + Number(rfc850[4]);
126
+ if (year > new Date(now).getUTCFullYear() + 50)
127
+ year -= 100;
128
+ return utcHttpDate(rfc850[1], rfc850[2], rfc850[3], year, rfc850[5], rfc850[6], rfc850[7]);
129
+ }
130
+ const asctime = ASCTIME_DATE.exec(value);
131
+ return asctime === null
132
+ ? null
133
+ : utcHttpDate(asctime[1], asctime[3] ?? asctime[4], asctime[2], Number(asctime[8]), asctime[5], asctime[6], asctime[7]);
134
+ };
135
+ // The package-shipped model teaching is loaded verbatim and fails hard if absent.
136
+ // The relative path is identical from src/ during development and dist/ after build.
42
137
  const documentation = await readFile(new URL("../docs/http.md", import.meta.url), "utf-8");
43
138
  class Http {
44
139
  static manifest = {
45
140
  name: "http",
46
141
  // Channel mimetypes here are SEED DEFAULTS (pre-fetch placeholders).
47
142
  // body is retyped per-call via notifyChunk's mimetype arg — to the real
48
- // response Content-Type, or projected markdown for a rendered page; octet-stream
49
- // is the honest "unknown until fetched". header is always the status
143
+ // response Content-Type or the configured readable projection type;
144
+ // octet-stream is the honest "unknown until fetched". header is always the status
50
145
  // line + headers (text/plain).
51
146
  channels: { [BODY]: "application/octet-stream", [HEADER]: "text/plain", html: "text/html" },
52
147
  defaultChannel: BODY,
53
148
  category: "data",
54
- scope: "workspace",
55
149
  writableBy: ["model", "client"],
56
150
  volatile: true, // remote content can change between fetches
57
151
  modelVisible: true,
@@ -62,318 +156,493 @@ class Http {
62
156
  requiresWeb: true, // excluded under the loop's noWeb flag
63
157
  },
64
158
  };
65
- // The render foundation (lazy chromium). Injectable for tests; one warm
66
- // pool per Http instance, shared across this scheme's fetches.
67
- #browser;
159
+ #errorDetailLimit;
68
160
  #webFetcher;
69
- constructor(browser = new Browser()) {
70
- this.#browser = browser;
71
- this.#webFetcher = new WebFetcher(browser);
161
+ constructor() {
162
+ this.#errorDetailLimit = ErrorDetail.configuredLimit();
163
+ this.#webFetcher = new WebFetcher();
72
164
  }
73
165
  async ready() {
74
- const mode = await this.#browser.ready?.();
75
- if (mode !== undefined)
76
- console.info(`http renderer: ${mode}`);
77
- }
78
- async close() {
79
- await this.#webFetcher.close();
80
- }
81
- // FIND itself is the consumer's standard entry query. This hook owns only
82
- // HTTP's prerequisite: an exact URL that is not already an entry must be
83
- // acquired through the same guarded byte/render path used by search
84
- // prefetch. Glob and regex targets only query already-known web entries.
85
- async prepareFind(statement, ctx) {
86
- const target = statement.target;
87
- if (target === null || target.kind !== "url")
88
- return { shape: "passthrough", status: 200 };
89
- const pathname = _a.#pathname(target);
166
+ WebFetcher.validateConfiguration();
167
+ }
168
+ // Representation preparation is scope-blind. Core applies the authored READ
169
+ // coordinates only after this hook has either materialized a finite GET or
170
+ // returned 102 for a retained SSE response.
171
+ async prepareRepresentation(request, ctx) {
172
+ if (request.target.kind !== "url") {
173
+ return _a.#bad(400, "http", "bad-target", "READ requires an http(s):// URL target.", {
174
+ stage: "target-validation",
175
+ recovery: "Provide an http(s):// URL target.",
176
+ retryable: false,
177
+ });
178
+ }
179
+ return this.#prepareGet(request.target, request.pathname, ctx);
180
+ }
181
+ async #prepareGet(target, pathname, ctx) {
182
+ const address = _a.#address(target);
183
+ if (!(address instanceof NetworkAddress))
184
+ return address;
185
+ const { url } = address;
186
+ const requestHeaders = target.headers ?? [];
187
+ let cached;
188
+ const conditional = [];
90
189
  const prior = await ctx.entries.read(pathname);
91
- if (prior.entry !== null)
92
- return { shape: "passthrough", status: 200 };
93
- const fetched = await this.#webFetcher.fetch(_a.#urlFrom(target), { signal: ctx.signal });
94
- if (fetched === null) {
95
- return _a.#bad(404, "http", "not_materialized", "the requested URL could not be materialized");
190
+ if (Results.isErrorStatus(prior.status) && prior.status !== 404) {
191
+ return _a.#passthrough(prior);
96
192
  }
97
- let { body, mimetype } = fetched;
98
- let channels = {
99
- [BODY]: { content: body, mimetype },
100
- ...(fetched.header === undefined ? {} : { [HEADER]: { content: fetched.header, mimetype: "text/plain" } }),
101
- };
102
- if (mimetype === "text/html") {
103
- let projected = await ctx.projection.readable(body, mimetype);
104
- if ((projected === null || projected.content.length === 0) && fetched.render !== undefined) {
105
- const rendered = await fetched.render();
106
- if (rendered !== null) {
107
- body = rendered.body;
108
- mimetype = rendered.mimetype;
109
- projected = await ctx.projection.readable(body, mimetype);
193
+ const priorBody = prior.entry?.channels[BODY];
194
+ const priorHeader = prior.entry?.channels[HEADER];
195
+ if (prior.entry !== null && _a.#requestMethod(priorHeader?.content ?? "") === undefined) {
196
+ return { status: 200 };
197
+ }
198
+ if (prior.entry !== null && priorBody !== undefined && priorHeader !== undefined) {
199
+ try {
200
+ if (await _a.#reusableGetRepresentation(prior.entry, requestHeaders, ctx.projection)) {
201
+ cached = prior.entry;
202
+ if (_a.#materializerIdentity(priorHeader.content) === undefined) {
203
+ conditional.push(..._a.#validators(priorHeader.content));
204
+ }
110
205
  }
111
206
  }
112
- if (projected === null || projected.content.length === 0) {
113
- return _a.#bad(422, "http", "no_readable_projection", "the requested URL has no readable projection");
207
+ catch (cause) {
208
+ return _a.#materializationFailure(url, "GET", new WebMaterializationError(_a.#sourceMimetype(priorHeader.content), cause));
114
209
  }
115
- channels = {
116
- [BODY]: { content: projected.content, mimetype: projected.mimetype },
117
- html: { content: body, mimetype },
118
- ...(fetched.header === undefined ? {} : { [HEADER]: { content: fetched.header, mimetype: "text/plain" } }),
119
- };
120
210
  }
121
- const written = await ctx.entries.write(pathname, { channels, tags: [] });
122
- return { shape: "passthrough", status: written.status };
123
- }
124
- // An unscoped READ fetches/revalidates. A scoped READ observes the already
125
- // materialized readable entry: refetching would discard the requested range
126
- // and return another asynchronous whole-body 102.
127
- async read(statement, ctx) {
128
- if (statement.target === null || statement.target.kind !== "url") {
129
- return _a.#bad(400, "http", "bad_target", "READ requires an http(s):// URL target");
211
+ if (cached !== undefined && _a.#fresh(cached.channels[HEADER].content)) {
212
+ return { status: 200 };
130
213
  }
131
- if (statement.lineMarker !== null) {
132
- const prior = await ctx.entries.read(_a.#pathname(statement.target));
133
- const body = prior.entry?.channels[BODY];
134
- if (body === undefined || body.content.length === 0) {
135
- return _a.#bad(409, "http", "scope_requires_materialization", "scoped READ requires a materialized response; READ the URL without <scope> first");
214
+ let fetched;
215
+ try {
216
+ fetched = await this.#webFetcher.fetch(url, {
217
+ signal: ctx.signal,
218
+ headers: requestHeaders,
219
+ ...(conditional.length > 0
220
+ ? { conditionalHeaders: conditional }
221
+ : {}),
222
+ guarded: false,
223
+ acceptHttpErrors: true,
224
+ preserveUnavailable: true,
225
+ });
226
+ }
227
+ catch (error) {
228
+ if (ctx.signal?.aborted === true && error === ctx.signal.reason) {
229
+ return _a.#cancelled(url, "GET");
136
230
  }
137
- const { error, ...read } = await ctx.entries.operations.read(statement);
138
- if (read.status >= 400) {
139
- return _a.#bad(read.status, "http", "scoped_read_failed", error ?? read.reason ?? `scoped READ failed with status ${read.status}`);
231
+ throw error;
232
+ }
233
+ if (fetched === null) {
234
+ return _a.#bad(404, "http", "not-materialized", `The URL ${url} could not be materialized.`, {
235
+ target: url,
236
+ stage: "acquisition",
237
+ retryable: true,
238
+ });
239
+ }
240
+ if (fetched.status === 304) {
241
+ if (cached === undefined) {
242
+ return _a.#bad(502, "http", "fetch-failed", `HTTP GET ${url} returned 304 without a reusable stored representation.`, {
243
+ target: url,
244
+ method: "GET",
245
+ stage: "acquisition",
246
+ retryable: true,
247
+ });
248
+ }
249
+ const cachedHeader = cached.channels[HEADER].content;
250
+ if (_a.#materializerIdentity(cachedHeader) !== undefined) {
251
+ return _a.#bad(502, "http", "fetch-failed", `HTTP GET ${url} returned 304 for a stored representation that requires full reacquisition.`, {
252
+ target: url,
253
+ method: "GET",
254
+ stage: "acquisition",
255
+ retryable: true,
256
+ });
257
+ }
258
+ const responseHeaders = fetched.responseHeaders ?? [];
259
+ if (!_a.#revalidationCorresponds(cachedHeader, new Headers(responseHeaders.map(([name, value]) => [name, value])))) {
260
+ return _a.#bad(502, "http", "fetch-failed", `HTTP GET ${url} returned 304 without identifying the stored representation nominated for revalidation.`, {
261
+ target: url,
262
+ method: "GET",
263
+ stage: "acquisition",
264
+ retryable: true,
265
+ });
266
+ }
267
+ const channels = {
268
+ ...cached.channels,
269
+ [HEADER]: {
270
+ ...cached.channels[HEADER],
271
+ content: _a.#refreshAfter304(cachedHeader, responseHeaders, requestHeaders),
272
+ },
273
+ };
274
+ const written = await ctx.entries.write(pathname, {
275
+ channels,
276
+ tags: cached.tags,
277
+ });
278
+ if (Results.isErrorStatus(written.status))
279
+ return _a.#passthrough(written);
280
+ return { status: 200 };
281
+ }
282
+ if (fetched.mimetype === "text/event-stream"
283
+ && fetched.response?.body !== null
284
+ && fetched.response?.body !== undefined) {
285
+ return this.#openEventStream(address, fetched, ctx);
286
+ }
287
+ let materialized;
288
+ try {
289
+ materialized = await WebFetcher.materialize(fetched, ctx.projection, ctx.signal);
290
+ }
291
+ catch (error) {
292
+ if (ctx.signal?.aborted === true)
293
+ return _a.#cancelled(url, "GET");
294
+ if (error instanceof WebMaterializationError) {
295
+ return _a.#materializationFailure(url, "GET", error);
140
296
  }
141
- return { shape: "passthrough", ...read };
297
+ throw error;
142
298
  }
143
- return this.#fetchStream(statement.target, ctx, "GET", undefined);
299
+ if (materialized === null) {
300
+ return _a.#bad(415, "http", "binary-response-unsupported", `HTTP GET ${url} returned ${fetched.mimetype}. The remote response was received, but its binary body cannot be represented in a Plurnk text channel.`, {
301
+ target: url,
302
+ method: "GET",
303
+ mimetype: fetched.mimetype,
304
+ stage: "materialization",
305
+ recovery: "Inspect #header or use a byte-capable client.",
306
+ retryable: false,
307
+ });
308
+ }
309
+ const written = await ctx.entries.write(pathname, {
310
+ channels: WebFetcher.materializedChannels(materialized, { url, method: "GET" }),
311
+ tags: [],
312
+ });
313
+ if (Results.isErrorStatus(written.status))
314
+ return _a.#passthrough(written);
315
+ return { status: 200 };
144
316
  }
145
- // EDIT → PUT the body (full-resource replace). `<L>` has no meaning against a
146
- // remote resource — reject rather than silently ignore the model's intent.
317
+ async #openEventStream(address, fetched, ctx) {
318
+ const response = fetched.response;
319
+ if (response?.body === null || response?.body === undefined) {
320
+ throw new Error("SSE preparation lost its response body.");
321
+ }
322
+ const local = new AbortController();
323
+ const handle = {
324
+ cancel: async () => {
325
+ local.abort();
326
+ await response.body?.cancel().catch(() => { });
327
+ },
328
+ };
329
+ const written = await ctx.entries.write(address.pathname, _a.#seedEntry());
330
+ if (Results.isErrorStatus(written.status))
331
+ return _a.#passthrough(written);
332
+ const subscription = await ctx.subscriptions.open(address.pathname, handle);
333
+ if (fetched.header !== undefined) {
334
+ await subscription.notifyChunk(HEADER, fetched.header, "text/plain");
335
+ }
336
+ void _a.#settleEventStream(subscription, response, {
337
+ url: address.url,
338
+ method: "GET",
339
+ signal: local.signal,
340
+ errorDetailLimit: this.#errorDetailLimit,
341
+ }).catch((error) => {
342
+ console.error("HTTP SSE terminal cleanup failed", { url: address.url, error });
343
+ });
344
+ return { shape: "passthrough", status: 102 };
345
+ }
346
+ // EDIT -> PUT the body (full-resource replace). `<L>` has no meaning against a
347
+ // remote resource - reject rather than silently ignore the model's intent.
147
348
  async editBatch(statements, ctx) {
148
349
  if (statements.length !== 1) {
149
- return _a.#bad(409, "http", "non_atomic_edit_batch", "multiple EDITs of one remote resource cannot be committed atomically");
350
+ return _a.#bad(409, "http", "non-atomic-edit-batch", `${statements.length} EDIT statements cannot be committed atomically to one remote resource.`, {
351
+ statements: statements.length,
352
+ stage: "mutation",
353
+ recovery: "Submit one whole-resource EDIT.",
354
+ retryable: false,
355
+ });
150
356
  }
151
357
  const statement = statements[0];
152
358
  if (statement.target === null || statement.target.kind !== "url") {
153
- return _a.#bad(400, "http", "bad_target", "EDIT requires an http(s):// URL target");
359
+ return _a.#bad(400, "http", "bad-target", "EDIT requires an http(s):// URL target.", {
360
+ stage: "target-validation",
361
+ recovery: "Provide an http(s):// URL target.",
362
+ retryable: false,
363
+ });
154
364
  }
155
365
  if (statement.lineMarker !== null) {
156
- return _a.#bad(400, "http", "no_line_edit", "EDIT on http PUTs the whole body; <L> line-editing a remote resource is unsupported");
366
+ return _a.#bad(400, "http", "line-edit-unsupported", "HTTP EDIT replaces the whole remote resource and does not accept a line range.", {
367
+ stage: "mutation",
368
+ recovery: "Remove the line range and submit the complete replacement body.",
369
+ retryable: false,
370
+ });
157
371
  }
158
- return this.#fetchStream(statement.target, ctx, "PUT", statement.body ?? "");
372
+ return this.#request(statement.target, ctx, "PUT", statement.body ?? "");
159
373
  }
160
374
  async edit(statement, ctx) {
161
375
  return this.editBatch([statement], ctx);
162
376
  }
163
- // KILL → DELETE the resource. Distinct from SEND[410] (which drops the local
377
+ // KILL -> DELETE the resource. Distinct from SEND[410] (which drops the local
164
378
  // cached entry): KILL is an HTTP DELETE request to the remote.
165
379
  async kill(statement, ctx) {
166
380
  if (statement.target === null || statement.target.kind !== "url") {
167
- return _a.#bad(400, "http", "bad_target", "KILL requires an http(s):// URL target");
381
+ return _a.#bad(400, "http", "bad-target", "KILL requires an http(s):// URL target.", {
382
+ stage: "target-validation",
383
+ recovery: "Provide an http(s):// URL target.",
384
+ retryable: false,
385
+ });
168
386
  }
169
- return this.#fetchStream(statement.target, ctx, "DELETE", statement.body ?? undefined);
387
+ return this.#request(statement.target, ctx, "DELETE", statement.body ?? undefined);
170
388
  }
171
- // SEND dispatch — status-code-as-verb (SPEC §op-surface).
172
- // 200 → request with body (POST), stream response
173
- // 410 → delete the cached entry
174
- // 499 → cancel in-flight (handled by the subscription's force-cancel;
389
+ // SEND dispatch — status-code-as-verb (SPEC {§op-surface}).
390
+ // 200 -> request with body (POST), stream response
391
+ // 410 -> delete the cached entry
392
+ // 499 -> cancel in-flight (handled by the subscription's force-cancel;
175
393
  // the engine routes 499 to the registered SubscriptionHandle, so a
176
394
  // scheme-level no-op here is correct — teardown already happened)
177
395
  async send(statement, ctx) {
178
396
  if (statement.target === null || statement.target.kind !== "url") {
179
- return _a.#bad(400, "http", "bad_target", "SEND requires an http(s):// URL target");
397
+ return _a.#bad(400, "http", "bad-target", "SEND requires an http(s):// URL target.", {
398
+ stage: "target-validation",
399
+ recovery: "Provide an http(s):// URL target.",
400
+ retryable: false,
401
+ });
180
402
  }
181
403
  const status = statement.signal;
182
404
  if (status === 200) {
183
405
  const body = statement.body?.raw ?? "";
184
- return this.#fetchStream(statement.target, ctx, "POST", body);
406
+ return this.#request(statement.target, ctx, "POST", body);
185
407
  }
186
408
  if (status === 410) {
187
- const { status: delStatus } = await ctx.entries.delete(_a.#pathname(statement.target));
188
- return { shape: "passthrough", status: delStatus };
409
+ const address = _a.#address(statement.target);
410
+ if (!(address instanceof NetworkAddress))
411
+ return address;
412
+ return _a.#passthrough(await ctx.entries.delete(address.pathname));
189
413
  }
190
414
  if (status === 499) {
191
415
  // Cancellation is routed by the engine to the subscription's
192
- // SubscriptionHandle.cancel (registered in #fetchStream). Nothing
416
+ // SubscriptionHandle.cancel (registered by #request). Nothing
193
417
  // for the scheme to do at the op level.
194
418
  return { shape: "passthrough", status: 200 };
195
419
  }
196
420
  // Entry-bearing schemes return 501 for status codes they don't interpret.
197
- return _a.#bad(501, "http", "unsupported_send", `SEND[${status}] not supported by http`);
198
- }
199
- // The streaming core, shared by every verb. Opens the subscription
200
- // (registering the abort handle for SEND[499] routing), fetches, then EITHER
201
- // renders (a GET of an HTML page is re-acquired through the browser, its
202
- // readable projection becomes body, and its final DOM is archived) OR streams the raw bytes (every non-GET
203
- // response and every non-HTML body). Request headers from the target's `{…}`
204
- // blocks (grammar#46) ride into both the fetch and the render. Each chunk is
205
- // labelled with its real mimetype via notifyChunk. Settles via close().
206
- async #fetchStream(target, ctx, method, body) {
207
- const url = _a.#rewriteHostileHost(_a.#urlFrom(target));
208
- const pathname = _a.#pathname(target);
209
- const headers = target.headers ?? []; // [key,value][] — opaque to grammar, honored here
421
+ return _a.#bad(501, "http", "send-status-unsupported", `The HTTP scheme does not interpret SEND status ${status}.`, {
422
+ requestedStatus: status,
423
+ stage: "dispatch",
424
+ retryable: false,
425
+ });
426
+ }
427
+ // Mutation responses use the same entry/channel and subscription primitives
428
+ // as every live producer; GET acquisition belongs solely to
429
+ // prepareRepresentation.
430
+ async #request(target, ctx, method, body) {
431
+ const address = _a.#address(target);
432
+ if (!(address instanceof NetworkAddress))
433
+ return address;
434
+ const { url } = address;
435
+ const { pathname } = address;
436
+ const headers = target.headers ?? [];
210
437
  const publishedChannel = target.fragment ?? _a.manifest.defaultChannel;
211
438
  if (!(publishedChannel in _a.manifest.channels)) {
212
- return _a.#bad(400, "http", "unknown_channel", `no channel #${publishedChannel}; channels: ${Object.keys(_a.manifest.channels).join(", ")}`);
213
- }
214
- // Conditional revalidation (GET only, service#341): recover the prior
215
- // fetch's validators + body from THIS scheme's own stored entry (entries
216
- // cap, own namespace — sanctioned). If the origin answers 304, we re-serve
217
- // the cached body and skip the expensive render. Freshness is decided ONLY
218
- // by #storedCopyServable (SPEC §revalidation): within the operator TTL
219
- // window the stored copy serves with zero round-trips; past it, the
220
- // conditional GET revalidates (service#333/#405 landed the TTL). Captured
221
- // BEFORE the seed write below overwrites the entry.
222
- let cached;
223
- const conditional = [];
224
- if (method === "GET") {
225
- const prior = await ctx.entries.read(pathname);
226
- const pb = prior.entry?.channels[BODY];
227
- if (pb !== undefined && pb.content.length > 0) {
228
- const ph = prior.entry?.channels[HEADER]?.content ?? "";
229
- const html = prior.entry?.channels.html;
230
- cached = {
231
- header: ph,
232
- body: { content: pb.content, mimetype: pb.mimetype },
233
- ...(html === undefined || html.content.length === 0 ? {} : { html: { content: html.content, mimetype: html.mimetype } }),
234
- };
235
- conditional.push(..._a.#validators(ph));
236
- }
237
- }
238
- // TTL fast path (the pre-fetch phase of the ONE predicate): a stored copy
239
- // still inside the freshness window serves with NO round-trip at all —
240
- // identical for a model READ and lane-1 prefetch (#405). Stamp unchanged:
241
- // the clock only resets when the ORIGIN vouches (fetch or 304), never by
242
- // serving from cache.
243
- if (cached !== undefined && _a.#storedCopyServable(_a.#fetchedAt(cached.header), null)) {
244
- await ctx.entries.write(pathname, _a.#seedEntry());
245
- await ctx.subscriptions.open(pathname, { cancel: () => { } }, { publishedChannel });
246
- await ctx.subscriptions.notifyChunk(HEADER, cached.header, "text/plain");
247
- if (cached.html !== undefined)
248
- await ctx.subscriptions.notifyChunk("html", cached.html.content, cached.html.mimetype);
249
- await ctx.subscriptions.notifyChunk(BODY, cached.body.content, cached.body.mimetype);
250
- await ctx.subscriptions.close("done", `ttl-fresh; ${cached.body.content.length} chars from cache`);
251
- return { shape: "passthrough", status: 102 };
439
+ const availableChannels = Object.keys(_a.manifest.channels);
440
+ return _a.#bad(400, "http", "channel-not-found", `Channel #${publishedChannel} does not exist on HTTP responses.`, {
441
+ requestedChannel: publishedChannel,
442
+ availableChannels,
443
+ recovery: `Use one of the available channels: ${availableChannels.map((channel) => `#${channel}`).join(", ")}.`,
444
+ retryable: false,
445
+ });
252
446
  }
253
447
  // Local AbortController for force-cancel from outside (SEND[499]).
254
448
  const local = new AbortController();
255
449
  const handle = { cancel: () => local.abort() };
256
- // Materialize the streaming target BEFORE subscribing (http#3). open()
257
- // binds an EXISTING entry — only the scheme knows its channel shape, so
258
- // it seeds them. Mirror exec's create-then-subscribe: write a seed entry
259
- // whose channels are the manifest's (body: octet-stream placeholder,
260
- // header: text/plain) — the same channels notifyChunk then populates.
261
- await ctx.entries.write(pathname, _a.#seedEntry());
450
+ // {§http-lifecycle} open() binds an existing entry, so the handler seeds
451
+ // its manifest-owned channel shape before subscribing.
452
+ const written = await ctx.entries.write(pathname, _a.#seedEntry());
453
+ if (Results.isErrorStatus(written.status))
454
+ return _a.#passthrough(written);
262
455
  // open() returns the worker+teardown-composed signal — fires on loop.cancel
263
- // OR our local teardown. Wire it so either path aborts the fetch/render.
264
- const composed = await ctx.subscriptions.open(pathname, handle, { publishedChannel });
456
+ // OR our local teardown. Wire it so either path aborts acquisition.
457
+ const subscription = await ctx.subscriptions.open(pathname, handle);
265
458
  const onAbort = () => local.abort();
266
- composed.addEventListener("abort", onAbort, { once: true });
459
+ subscription.addEventListener("abort", onAbort, { once: true });
267
460
  try {
268
461
  const response = await fetch(url, {
269
462
  method,
270
463
  body,
271
- // One browser identity on the wire (never Node's default "node"
272
- // UA — a loud automated-client fingerprint). The model's own
273
- // {User-Agent: …} block wins when present.
274
464
  headers: headers.some(([k]) => k.toLowerCase() === "user-agent")
275
- ? [...headers, ...conditional]
276
- : [["User-Agent", BROWSER_UA], ...headers, ...conditional],
465
+ ? headers
466
+ : [["User-Agent", DEFAULT_WEB_UA], ...headers],
277
467
  signal: local.signal,
278
468
  redirect: "follow",
279
469
  });
280
- // Origin confirms the cached copy is current → re-serve it, skip the
281
- // render/stream. A 304-serve is a first-class READ: the model sees the
282
- // stored body as an ordinary streaming result, never a cache status
283
- // (service#341). `revalidated 304` rides the close summary for the log
284
- // meta line + digest. notifyChunk appends onto the freshly-seeded empty
285
- // channels, restoring the cached header + body.
286
- if (cached !== undefined && _a.#storedCopyServable(_a.#fetchedAt(cached.header), response)) {
287
- await ctx.subscriptions.notifyChunk(HEADER, _a.#stamp(cached.header), "text/plain");
288
- if (cached.html !== undefined)
289
- await ctx.subscriptions.notifyChunk("html", cached.html.content, cached.html.mimetype);
290
- await ctx.subscriptions.notifyChunk(BODY, cached.body.content, cached.body.mimetype);
291
- await ctx.subscriptions.close("done", `revalidated 304; ${cached.body.content.length} chars from cache`);
292
- return { shape: "passthrough", status: 102 };
293
- }
294
- const contentType = response.headers.get("content-type") ?? "";
295
- // Always-render: a GET of an HTML page is re-acquired through the
296
- // browser so the body is projected from the final rendered DOM. The probe-fetch
297
- // body is discarded — the browser does its own navigation. Only GET
298
- // renders; POST/PUT/DELETE can't be replayed as a browser navigation.
299
- const isHtml = method === "GET"
300
- && /^(?:text\/html|application\/xhtml\+xml)\b/i.test(contentType);
301
- if (isHtml) {
302
- await response.body?.cancel();
303
- const result = await this.#browser.render(url, { workerId: ctx.workerId, signal: local.signal, headers });
304
- const projected = await ctx.projection.readable(result.html, "text/html");
305
- if (projected === null)
306
- throw new Error("rendered HTML has no readable projection");
307
- await _a.#writeHeader(ctx, result.status, result.statusText, result.headers);
308
- await ctx.subscriptions.notifyChunk("html", result.html, "text/html");
309
- await ctx.subscriptions.notifyChunk(BODY, projected.content, projected.mimetype);
310
- await ctx.subscriptions.close("done", `rendered HTTP ${result.status}; ${projected.content.length} readable chars`);
311
- return { shape: "passthrough", status: 102 };
312
- }
313
- // SSE: an event stream is parsed into its `data` payloads (SPEC §sse),
314
- // one notifyChunk per event with the `data:`/comment framing stripped,
315
- // so the model reads event content and not the transport. A long-lived
316
- // GET; events land in the body channel as they arrive across turns,
317
- // until the origin closes. Only GET — a POST reply is never an SSE READ.
318
- if (method === "GET" && /^text\/event-stream\b/i.test(contentType)) {
319
- await _a.#writeHeader(ctx, response.status, response.statusText, [...response.headers]);
320
- return await _a.#streamEvents(ctx, response);
321
- }
322
- // Byte path: stream the body labelled with its real content type.
323
- await _a.#writeHeader(ctx, response.status, response.statusText, [...response.headers]);
324
- const bodyMime = contentType.split(";")[0].trim() || "application/octet-stream";
470
+ const responseMime = responseMimetype(response.headers.get("content-type"));
471
+ // {§http-lifecycle}/{§mimetype-classifier} String channels retain
472
+ // textual response data. Binary input is transient: an installed
473
+ // reader may derive Unicode, otherwise the durable body is a typed
474
+ // empty marker rather than a fabricated byte channel.
475
+ await _a.#writeHeader(subscription, method, response.status, response.statusText, [...response.headers], headers);
476
+ const bodyMime = responseMime;
325
477
  if (response.body === null) {
326
- await ctx.subscriptions.close("done", `HTTP ${response.status}; empty body`);
478
+ await subscription.close({ status: 200 }, `HTTP ${response.status}; empty body`);
327
479
  return { shape: "passthrough", status: 102 };
328
480
  }
481
+ const responseBody = response.body;
482
+ const byteBody = {
483
+ chunks: responseBody,
484
+ cancel: () => responseBody.cancel(),
485
+ };
486
+ const binary = await WebFetcher.classifyBinary(byteBody, bodyMime, ctx.projection);
487
+ if (binary) {
488
+ let projected;
489
+ try {
490
+ projected = await WebFetcher.projectBytes(byteBody, bodyMime, ctx.projection);
491
+ }
492
+ catch (error) {
493
+ await subscription.notifyChunk(BODY, "", bodyMime);
494
+ throw error;
495
+ }
496
+ if (projected !== null) {
497
+ await _a.#writeProjectionIdentity(subscription, projected.projectionIdentity);
498
+ await subscription.notifyChunk(BODY, projected.content, projected.mimetype);
499
+ await subscription.close({ status: 200 }, `HTTP ${response.status}; ${projected.content.length} readable chars from ${bodyMime}`);
500
+ return { shape: "passthrough", status: 102 };
501
+ }
502
+ await subscription.notifyChunk(BODY, "", bodyMime);
503
+ const detail = `HTTP ${method} ${url} returned ${bodyMime}. The remote response was received, but its binary body cannot be represented in a Plurnk text channel.`;
504
+ const result = _a.#bad(415, "http", "binary-response-unsupported", detail, {
505
+ target: url,
506
+ method,
507
+ mimetype: bodyMime,
508
+ stage: "materialization",
509
+ recovery: "Do not retry the request solely to retrieve this body; inspect #header or use a byte-capable client.",
510
+ retryable: false,
511
+ });
512
+ await subscription.close(result, detail);
513
+ return result;
514
+ }
515
+ // {§http-text-decoding} Fetch text is replacement-mode UTF-8;
516
+ // Content-Type charset remains response evidence, not a second decoder.
329
517
  let bytes = 0;
330
518
  const decoder = new TextDecoder();
331
- for await (const chunk of response.body) {
519
+ for await (const chunk of responseBody) {
332
520
  bytes += chunk.length;
333
- await ctx.subscriptions.notifyChunk(BODY, decoder.decode(chunk, { stream: true }), bodyMime);
521
+ await subscription.notifyChunk(BODY, decoder.decode(chunk, { stream: true }), bodyMime);
334
522
  }
335
523
  const tail = decoder.decode();
336
524
  if (tail.length > 0)
337
- await ctx.subscriptions.notifyChunk(BODY, tail, bodyMime);
338
- await ctx.subscriptions.close("done", `HTTP ${response.status}; ${bytes} bytes`);
525
+ await subscription.notifyChunk(BODY, tail, bodyMime);
526
+ await subscription.close({ status: 200 }, `HTTP ${response.status}; ${bytes} bytes`);
339
527
  return { shape: "passthrough", status: 102 };
340
528
  }
341
529
  catch (err) {
342
530
  const aborted = local.signal.aborted;
343
- const reason = aborted ? "aborted" : err instanceof Error ? err.message : String(err);
344
- await ctx.subscriptions.close(aborted ? "cancelled" : "error", reason);
345
- // 499 for client-cancelled, 502 for upstream/network/render failure.
346
- return _a.#bad(aborted ? 499 : 502, "http", aborted ? "aborted" : "fetch_failed", reason);
531
+ if (aborted) {
532
+ const result = _a.#cancelled(url, method);
533
+ await subscription.close(result, result.problem?.detail);
534
+ return result;
535
+ }
536
+ if (err instanceof WebMaterializationError) {
537
+ const result = _a.#materializationFailure(url, method, err);
538
+ await subscription.close(result, result.problem?.detail);
539
+ return result;
540
+ }
541
+ console.error("HTTP acquisition failed", { method, url, err });
542
+ const cause = ErrorDetail.preview(err, this.#errorDetailLimit);
543
+ const reason = `HTTP ${method} ${url} failed: ${cause}`;
544
+ // The remaining catch owns acquisition failure; cancellation and
545
+ // typed materialization failures settled above.
546
+ const result = _a.#bad(502, "http", "fetch-failed", reason, {
547
+ target: url,
548
+ method,
549
+ stage: "acquisition",
550
+ retryable: true,
551
+ });
552
+ await subscription.close(result, reason);
553
+ return result;
347
554
  }
348
555
  finally {
349
- composed.removeEventListener("abort", onAbort);
556
+ subscription.removeEventListener("abort", onAbort);
350
557
  }
351
558
  }
352
- // Seed entry mirroring the manifest's channels — empty content + the seed
353
- // mimetypes (body: octet-stream until the fetch retypes it via notifyChunk,
354
- // header: text/plain). This is the channel-shape knowledge open() lacks; the
355
- // scheme materializes the target so the subscription binds an existing entry
356
- // (http#3). Fresh stream target → no tags.
559
+ // {§http-manifest}/{§http-lifecycle} Seed the declared channel shape before
560
+ // open(); notifyChunk later welds acquired content to its actual mimetype.
357
561
  static #seedEntry() {
358
- const channels = Object.fromEntries(Object.entries(_a.manifest.channels).map(([name, mimetype]) => [name, { content: "", mimetype }]));
562
+ const channels = Object.fromEntries(Object.entries(_a.manifest.channels).map(([name, mimetype]) => [name, {
563
+ content: "",
564
+ mimetype,
565
+ state: "active",
566
+ }]));
359
567
  return { channels, tags: [] };
360
568
  }
361
- // Record the response status line + headers into the HEADER channel (text/plain).
362
- static async #writeHeader(ctx, status, statusText, headers) {
569
+ // {§revalidation} The body and header must have settled successfully. An
570
+ // auxiliary channel may have terminally failed without invalidating body.
571
+ static #representationComplete(entry) {
572
+ const bodyState = entry.channels[BODY]?.state;
573
+ const headerState = entry.channels[HEADER]?.state;
574
+ const successful = (state) => state === "static" || state === "closed";
575
+ const terminal = (state) => successful(state) || state === "errored";
576
+ return successful(bodyState)
577
+ && successful(headerState)
578
+ && Object.values(entry.channels).every(({ state }) => terminal(state));
579
+ }
580
+ // {§revalidation} Direct READ and exact FIND share the complete reusable
581
+ // representation predicate; freshness only decides how READ uses a match.
582
+ static async #reusableGetRepresentation(entry, requestHeaders, projection) {
583
+ const body = entry.channels[BODY];
584
+ const header = entry.channels[HEADER];
585
+ if (body === undefined || header === undefined)
586
+ return false;
587
+ return _a.#representationComplete(entry)
588
+ && _a.#requestMethod(header.content) === "GET"
589
+ && requestHeaders.length === 0
590
+ && _a.#cacheVariant(header.content) === "default"
591
+ && !_a.#storedCachePolicy(header.content).noStore
592
+ && await _a.#projectionCurrent(header.content, projection);
593
+ }
594
+ // {§revalidation} Origin headers come first; authoritative package method
595
+ // and acquisition metadata are appended last.
596
+ static async #writeHeader(subscription, method, status, statusText, responseHeaders, requestHeaders) {
597
+ await subscription.notifyChunk(HEADER, _a.#responseHeader(method, status, statusText, responseHeaders, requestHeaders), "text/plain");
598
+ }
599
+ static #responseHeader(method, status, statusText, responseHeaders, requestHeaders) {
363
600
  const lines = [`HTTP ${status} ${statusText}`];
364
- for (const [k, v] of headers)
601
+ for (const [k, v] of responseHeaders)
365
602
  lines.push(`${k}: ${v}`);
603
+ lines.push(`${REQUEST_METHOD}: ${method}`);
366
604
  lines.push(`${FETCHED_AT}: ${new Date().toISOString()}`);
367
- await ctx.subscriptions.notifyChunk(HEADER, lines.join("\n"), "text/plain");
368
- }
369
- // Drain an SSE body through the standard streaming parser and dispatch one
370
- // BODY chunk per event. event/id/retry/comments remain transport metadata:
371
- // this projection publishes data, not the wire, and does not reconnect.
372
- static async #streamEvents(ctx, response) {
373
- if (response.body === null) {
374
- await ctx.subscriptions.close("done", "SSE stream; empty body");
375
- return { shape: "passthrough", status: 102 };
605
+ lines.push(cacheVariantEvidence(classifyCacheVariant(requestHeaders, responseHeaders)));
606
+ return lines.join("\n");
607
+ }
608
+ static async #writeProjectionIdentity(subscription, identity) {
609
+ await subscription.notifyChunk(HEADER, `\n${WebFetcher.projectionEvidence(identity)}`, "text/plain");
610
+ }
611
+ // Drain an acquired SSE body and settle its retained subscription. The READ
612
+ // has already returned 102, so terminal failure is durable stream state.
613
+ static async #settleEventStream(subscription, response, options) {
614
+ try {
615
+ const events = await _a.#streamEvents(subscription, response);
616
+ if (options.signal.aborted) {
617
+ const result = _a.#cancelled(options.url, options.method);
618
+ await subscription.close(result, result.problem?.detail);
619
+ return;
620
+ }
621
+ await subscription.close({ status: 200 }, `SSE stream; ${events} events`);
622
+ }
623
+ catch (error) {
624
+ if (options.signal.aborted) {
625
+ const result = _a.#cancelled(options.url, options.method);
626
+ await subscription.close(result, result.problem?.detail);
627
+ return;
628
+ }
629
+ console.error("HTTP SSE stream failed", { method: options.method, url: options.url, error });
630
+ const cause = ErrorDetail.preview(error, options.errorDetailLimit);
631
+ const reason = `HTTP ${options.method} ${options.url} failed: ${cause}`;
632
+ const result = _a.#bad(502, "http", "fetch-failed", reason, {
633
+ target: options.url,
634
+ method: options.method,
635
+ stage: "transfer",
636
+ retryable: true,
637
+ });
638
+ await subscription.close(result, reason);
376
639
  }
640
+ }
641
+ // event/id/retry/comments remain transport metadata: this projection
642
+ // publishes event data, not the wire, and does not reconnect.
643
+ static async #streamEvents(subscription, response) {
644
+ if (response.body === null)
645
+ return 0;
377
646
  const decoder = new TextDecoder();
378
647
  const pending = [];
379
648
  let fatal = null;
@@ -391,7 +660,7 @@ class Http {
391
660
  const publish = async () => {
392
661
  for (const data of pending.splice(0)) {
393
662
  events += 1;
394
- await ctx.subscriptions.notifyChunk(BODY, `${data}\n`, "text/plain");
663
+ await subscription.notifyChunk(BODY, `${data}\n`, "text/plain");
395
664
  }
396
665
  };
397
666
  for await (const chunk of response.body) {
@@ -407,89 +676,306 @@ class Http {
407
676
  if (fatal !== null)
408
677
  throw fatal;
409
678
  await publish();
410
- await ctx.subscriptions.close("done", `SSE stream; ${events} events`);
411
- return { shape: "passthrough", status: 102 };
679
+ return events;
412
680
  }
413
- // Reconstruct the absolute URL from the parsed UrlPath. `raw` is the
414
- // grammar's verbatim URL — authoritative; the decomposed fields are a
415
- // convenience. Use raw so query strings / auth / port survive exactly.
416
- static #urlFrom(target) {
417
- return target.raw;
418
- }
419
- // Entry storage has no authority column. The host is part of a web
420
- // resource's identity, so fold it into the pathname exactly as the search
421
- // prefetch sink does. Routing (`https` rides the HTTP handler) remains
422
- // separate from storage (`https` remains https).
423
- static #pathname(target) {
424
- const folded = target.hostname ? `/${target.hostname}${target.pathname}` : target.pathname;
425
- // `%28`/`%29` are the grammar-safe transport spelling of literal path
426
- // parentheses. Entry storage is canonical; the packet renderer encodes
427
- // them again when presenting an address to the model.
428
- return folded.replace(/%28/giu, "(").replace(/%29/giu, ")");
429
- }
430
- // Known-hostile-host rewrite — the ONE bounded, first-party exception
431
- // (SPEC §host-rewrite, schemes-http#4). A GitHub blob page is a CSP-locked JS SPA that renders
432
- // nothing useful, and code wants SOURCE (line-navigable) not markdown pulled
433
- // from a rendered code-viewer; raw.githubusercontent serves the exact bytes
434
- // on the byte path. Measured through the extractor: blob → SPA/JSON noise,
435
- // raw → clean source. The `{ref}/{path}` tail carries through verbatim, so
436
- // slash-bearing branch refs map correctly. Wikipedia was measured too and
437
- // deliberately gets NO rewrite — desktop already extracts to the full clean
438
- // article; every rewrite (action=render, mobile-html) regressed it.
439
- static #rewriteHostileHost(url) {
440
- const gh = /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/blob\/(.+)$/.exec(url);
441
- if (gh !== null)
442
- return `https://raw.githubusercontent.com/${gh[1]}/${gh[2]}/${gh[3]}`;
443
- return url;
444
- }
445
- // The single freshness-predicate boundary (service#341, #333). Today a
446
- // conditional-GET `304 Not Modified` proves the stored copy is current, so it
447
- // is servable without a re-fetch/re-render. service#333's per-URL TTL lands
448
- // HERE as the SAME predicate (a pre-fetch branch) — one stamp, one rule,
449
- // identical at every entry point. Never add a second freshness check elsewhere.
450
- // Pre-fetch phase (response === null): fresh iff the stamp is inside the
451
- // operator TTL window; PLURNK_SCHEMES_HTTP_TTL_MS=0 disables the window so
452
- // every read revalidates (the pre-TTL behavior). Post-fetch phase: a 304 is
453
- // the origin vouching for the stored copy.
454
- static #storedCopyServable(fetchedAtMs, response) {
455
- if (response === null) {
456
- const ttl = requireNumEnv("PLURNK_SCHEMES_HTTP_TTL_MS");
457
- return ttl > 0 && fetchedAtMs !== undefined && Date.now() - fetchedAtMs < ttl;
458
- }
459
- return response.status === 304;
460
- }
461
- // Parse the materialization stamp out of a stored HEADER channel; undefined
462
- // when absent (pre-TTL entries, execs-materialized pages) — those simply
463
- // never TTL-serve and revalidate instead.
681
+ static #address(target) {
682
+ let address;
683
+ try {
684
+ address = NetworkAddress.from(target);
685
+ }
686
+ catch {
687
+ return _a.#bad(400, "http", "bad-target", "HTTP operations require an http(s):// URL with an authority.", {
688
+ stage: "target-validation",
689
+ recovery: "Provide an http(s):// URL with a host.",
690
+ retryable: false,
691
+ });
692
+ }
693
+ if (address.scheme !== "http" && address.scheme !== "https") {
694
+ return _a.#bad(400, "http", "bad-target", "HTTP operations require an http(s):// URL target.", {
695
+ stage: "target-validation",
696
+ recovery: "Provide an http(s):// URL target.",
697
+ retryable: false,
698
+ });
699
+ }
700
+ if (address.hasCredentials) {
701
+ return _a.#bad(400, "http", "userinfo-not-allowed", "HTTP URL userinfo is not allowed.", {
702
+ target: address.url,
703
+ stage: "target-validation",
704
+ recovery: "Remove credentials from the URL and use request metadata where authorization is required.",
705
+ retryable: false,
706
+ });
707
+ }
708
+ return address;
709
+ }
710
+ // {§revalidation} Validation-free reuse requires both the operator's local
711
+ // ceiling and the origin's response policy to remain live.
712
+ static #fresh(header, now = Date.now()) {
713
+ const fetchedAt = _a.#fetchedAt(header);
714
+ const ttl = requireNumEnv("PLURNK_SCHEMES_HTTP_TTL_MS");
715
+ if (fetchedAt === undefined || ttl <= 0)
716
+ return false;
717
+ const residentTime = Math.max(0, now - fetchedAt);
718
+ if (residentTime >= ttl)
719
+ return false;
720
+ const policy = _a.#storedCachePolicy(header);
721
+ if (policy.noStore || policy.noCache)
722
+ return false;
723
+ return policy.freshnessLifetimeMs === undefined
724
+ || _a.#currentAge(header, fetchedAt, now) < policy.freshnessLifetimeMs;
725
+ }
726
+ // Package metadata is appended after origin headers, so the last value wins.
464
727
  static #fetchedAt(priorHeader) {
465
- const m = new RegExp(`^${FETCHED_AT}:[ \\t]*(.+)$`, "im").exec(priorHeader);
466
- if (m === null)
728
+ const value = lastHeaderValue(priorHeader, FETCHED_AT);
729
+ if (value === undefined)
467
730
  return undefined;
468
- const ms = Date.parse(m[1].trim());
731
+ const ms = Date.parse(value);
469
732
  return Number.isNaN(ms) ? undefined : ms;
470
733
  }
471
- // Re-stamp a stored HEADER to now — used when the ORIGIN vouches (304).
472
- static #stamp(header) {
473
- const line = `${FETCHED_AT}: ${new Date().toISOString()}`;
474
- const re = new RegExp(`^${FETCHED_AT}:.*$`, "im");
475
- return re.test(header) ? header.replace(re, line) : `${header}\n${line}`;
734
+ static #headerField(line) {
735
+ const colon = line.indexOf(":");
736
+ if (colon < 0)
737
+ return null;
738
+ const name = line.slice(0, colon).trim().toLowerCase();
739
+ if (name.length === 0)
740
+ return null;
741
+ return { name, value: line.slice(colon + 1).trim(), line };
742
+ }
743
+ static #packageStart(lines) {
744
+ let stampIndex = -1;
745
+ for (const [index, line] of lines.entries()) {
746
+ if (_a.#headerField(line)?.name === FETCHED_AT)
747
+ stampIndex = index;
748
+ }
749
+ if (stampIndex < 0)
750
+ return lines.length;
751
+ return _a.#headerField(lines[stampIndex - 1] ?? "")?.name === REQUEST_METHOD
752
+ ? stampIndex - 1
753
+ : stampIndex;
754
+ }
755
+ static #originFields(header) {
756
+ const lines = header.split(/\r?\n/);
757
+ return lines.slice(1, _a.#packageStart(lines))
758
+ .map((line) => _a.#headerField(line))
759
+ .filter((field) => field !== null);
760
+ }
761
+ static #originValues(fields, name) {
762
+ return fields.filter((field) => field.name === name).map(({ value }) => value);
763
+ }
764
+ static #storedCachePolicy(header) {
765
+ const fields = _a.#originFields(header);
766
+ const directives = _a.#originValues(fields, "cache-control")
767
+ .flatMap((value) => splitHttpList(value))
768
+ .map((member) => cacheDirective(member))
769
+ .filter((directive) => directive !== null);
770
+ const maxAges = directives.filter(({ name }) => name === "max-age");
771
+ let freshnessLifetimeMs;
772
+ if (maxAges.length > 0) {
773
+ freshnessLifetimeMs = maxAges.length === 1
774
+ ? deltaMilliseconds(maxAges[0].argument) ?? 0
775
+ : 0;
776
+ }
777
+ else {
778
+ const expiresValues = _a.#originValues(fields, "expires");
779
+ if (expiresValues.length > 0) {
780
+ const expires = expiresValues.length === 1
781
+ ? httpDate(expiresValues[0])
782
+ : null;
783
+ const fetchedAt = _a.#fetchedAt(header);
784
+ if (expires === null || fetchedAt === undefined) {
785
+ freshnessLifetimeMs = 0;
786
+ }
787
+ else {
788
+ const dateValues = _a.#originValues(fields, "date");
789
+ const parsedDate = dateValues.length === 1
790
+ ? httpDate(dateValues[0])
791
+ : null;
792
+ const generatedAt = parsedDate ?? fetchedAt;
793
+ freshnessLifetimeMs = Math.max(0, expires - generatedAt);
794
+ }
795
+ }
796
+ }
797
+ return {
798
+ noStore: directives.some(({ name }) => name === "no-store"),
799
+ noCache: directives.some(({ name }) => name === "no-cache"),
800
+ ...(freshnessLifetimeMs === undefined ? {} : { freshnessLifetimeMs }),
801
+ };
802
+ }
803
+ static #currentAge(header, fetchedAt, now) {
804
+ const fields = _a.#originFields(header);
805
+ const dateValues = _a.#originValues(fields, "date");
806
+ const date = dateValues.length === 1 ? httpDate(dateValues[0]) : null;
807
+ const apparentAge = date === null ? 0 : Math.max(0, fetchedAt - date);
808
+ const ageMember = _a.#originValues(fields, "age").flatMap(splitHttpList)[0];
809
+ const ageValue = deltaMilliseconds(ageMember) ?? 0;
810
+ return Math.max(apparentAge, ageValue) + Math.max(0, now - fetchedAt);
811
+ }
812
+ static #requestMethod(priorHeader) {
813
+ return lastHeaderValue(priorHeader, REQUEST_METHOD)?.toUpperCase();
814
+ }
815
+ // Package evidence is authoritative only after the package acquisition
816
+ // stamp. An origin field with the same name remains inert.
817
+ static #packageHeaderValue(priorHeader, field) {
818
+ const lines = priorHeader.split(/\r?\n/);
819
+ let stampIndex = -1;
820
+ for (const [index, line] of lines.entries()) {
821
+ const colon = line.indexOf(":");
822
+ if (colon < 0)
823
+ continue;
824
+ const name = line.slice(0, colon).trim().toLowerCase();
825
+ if (name === FETCHED_AT)
826
+ stampIndex = index;
827
+ }
828
+ if (stampIndex < 0)
829
+ return undefined;
830
+ let value;
831
+ for (const line of lines.slice(stampIndex + 1)) {
832
+ const colon = line.indexOf(":");
833
+ if (colon < 0 || line.slice(0, colon).trim().toLowerCase() !== field)
834
+ continue;
835
+ value = line.slice(colon + 1).trim();
836
+ }
837
+ return value === "" ? undefined : value;
838
+ }
839
+ static #projectionIdentity(priorHeader) {
840
+ return _a.#packageHeaderValue(priorHeader, PROJECTION_ID_HEADER);
841
+ }
842
+ static #materializerIdentity(priorHeader) {
843
+ return _a.#packageHeaderValue(priorHeader, MATERIALIZER_ID_HEADER);
844
+ }
845
+ static #cacheVariant(priorHeader) {
846
+ const value = _a.#packageHeaderValue(priorHeader, CACHE_VARIANT_HEADER);
847
+ return value === "default" || value === "bypass" ? value : undefined;
848
+ }
849
+ static #sourceMimetype(header) {
850
+ return responseMimetype(lastHeaderValue(header, "content-type") ?? null);
851
+ }
852
+ static async #projectionCurrent(header, projection) {
853
+ const sourceMimetype = _a.#sourceMimetype(header);
854
+ const isHtml = MimetypeClassifier.isHtml(sourceMimetype);
855
+ const materializerIdentity = _a.#materializerIdentity(header);
856
+ if (materializerIdentity !== undefined) {
857
+ if (!WebFetcher.materializerCurrent(materializerIdentity))
858
+ return false;
859
+ if (materializerIdentity.startsWith("tavily-extract:")
860
+ || materializerIdentity === "origin-markdown:v1")
861
+ return true;
862
+ }
863
+ if (isHtml && materializerIdentity === undefined)
864
+ return false;
865
+ const storedIdentity = _a.#projectionIdentity(header);
866
+ if (storedIdentity === undefined)
867
+ return !isHtml;
868
+ return await projection.identity(sourceMimetype) === storedIdentity;
869
+ }
870
+ // RFC 9111 §3.2 permits a processed cache to retain the metadata that its
871
+ // stored body depends upon. Other 304 fields update by name; package-owned
872
+ // evidence is then rebuilt after the origin block. {§revalidation}
873
+ static #refreshAfter304(header, responseHeaders, requestHeaders) {
874
+ const lines = header.split(/\r?\n/);
875
+ const origin = _a.#originFields(header);
876
+ const updates = responseHeaders.filter(([name]) => !BODY_PROCESSING_FIELDS.has(name.toLowerCase()));
877
+ const updatedNames = new Set(updates.map(([name]) => name.toLowerCase()));
878
+ const retained = origin.filter(({ name }) => !updatedNames.has(name));
879
+ const merged = [
880
+ ...retained.map(({ name, value }) => [name, value]),
881
+ ...updates,
882
+ ];
883
+ const packageEvidence = [
884
+ MATERIALIZER_ID_HEADER,
885
+ PROJECTION_ID_HEADER,
886
+ TAVILY_REQUEST_ID_HEADER,
887
+ TAVILY_CREDITS_HEADER,
888
+ ].flatMap((field) => {
889
+ const value = _a.#packageHeaderValue(header, field);
890
+ return value === undefined ? [] : [`${field}: ${value}`];
891
+ });
892
+ return [
893
+ lines[0] ?? "HTTP 200 OK",
894
+ ...retained.map(({ line }) => line),
895
+ ...updates.map(([name, value]) => `${name}: ${value}`),
896
+ `${REQUEST_METHOD}: GET`,
897
+ `${FETCHED_AT}: ${new Date().toISOString()}`,
898
+ cacheVariantEvidence(classifyCacheVariant(requestHeaders, merged)),
899
+ ...packageEvidence,
900
+ ].join("\n");
901
+ }
902
+ static #revalidationCorresponds(priorHeader, responseHeaders) {
903
+ const fields = _a.#originFields(priorHeader);
904
+ const responseTagValue = responseHeaders.get("etag");
905
+ if (responseTagValue !== null) {
906
+ const storedTagValues = _a.#originValues(fields, "etag");
907
+ const responseTag = entityTag(responseTagValue);
908
+ const storedTag = storedTagValues.length === 1
909
+ ? entityTag(storedTagValues[0])
910
+ : null;
911
+ if (responseTag === null || storedTag === null)
912
+ return false;
913
+ return responseTag.opaque === storedTag.opaque
914
+ && (responseTag.weak || !storedTag.weak);
915
+ }
916
+ const responseModifiedValue = responseHeaders.get("last-modified");
917
+ const storedModifiedValues = _a.#originValues(fields, "last-modified");
918
+ const responseModified = responseModifiedValue === null
919
+ ? null
920
+ : httpDate(responseModifiedValue);
921
+ const storedModified = storedModifiedValues.length === 1
922
+ ? httpDate(storedModifiedValues[0])
923
+ : null;
924
+ return responseModified !== null
925
+ && storedModified !== null
926
+ && responseModified === storedModified;
476
927
  }
477
928
  // Conditional-request headers from the prior fetch's stored response headers
478
929
  // (the HEADER channel text #writeHeader wrote): ETag → If-None-Match,
479
930
  // Last-Modified → If-Modified-Since. Empty when neither is present — the
480
931
  // origin then just 200s with a full body, which is correct.
481
932
  static #validators(priorHeader) {
933
+ const fields = _a.#originFields(priorHeader);
482
934
  const out = [];
483
- const etag = /^etag:[ \t]*(.+)$/im.exec(priorHeader);
484
- if (etag !== null)
485
- out.push(["If-None-Match", etag[1].trim()]);
486
- const lastModified = /^last-modified:[ \t]*(.+)$/im.exec(priorHeader);
487
- if (lastModified !== null)
488
- out.push(["If-Modified-Since", lastModified[1].trim()]);
935
+ const etags = _a.#originValues(fields, "etag");
936
+ if (etags.length === 1 && entityTag(etags[0]) !== null) {
937
+ out.push(["If-None-Match", etags[0]]);
938
+ }
939
+ const lastModified = _a.#originValues(fields, "last-modified");
940
+ if (lastModified.length === 1 && httpDate(lastModified[0]) !== null) {
941
+ out.push(["If-Modified-Since", lastModified[0]]);
942
+ }
489
943
  return out;
490
944
  }
491
- static #bad(status, scheme, kind, message) {
492
- return { shape: "passthrough", status, error: Results.error(scheme, kind, message) };
945
+ static #passthrough(result) {
946
+ return Results.assert({ ...result, shape: "passthrough" });
947
+ }
948
+ static #cancelled(url, method) {
949
+ return _a.#bad(499, "http", "cancelled", `HTTP ${method} ${url} was cancelled.`, {
950
+ target: url,
951
+ method,
952
+ stage: "acquisition",
953
+ retryable: false,
954
+ });
955
+ }
956
+ static #materializationFailure(url, method, error) {
957
+ if (error.stage === "projection" && error.cause instanceof ProjectionInputLimitError) {
958
+ return _a.#bad(413, "http", "projection-input-limit", `HTTP ${method} ${url} exceeded the ${error.cause.maximumBytes}-byte readable-projection input limit.`, {
959
+ target: url,
960
+ method,
961
+ mimetype: error.cause.mimetype,
962
+ maximumBytes: error.cause.maximumBytes,
963
+ observedBytes: error.cause.observedBytes,
964
+ stage: "projection",
965
+ retryable: false,
966
+ });
967
+ }
968
+ console.error("HTTP materialization failed", { method, url, error });
969
+ return _a.#bad(500, "http", "projection-failed", `HTTP ${method} ${url} acquired content, but its readable projection failed.`, {
970
+ target: url,
971
+ method,
972
+ mimetype: error.mimetype,
973
+ stage: "projection",
974
+ retryable: false,
975
+ });
976
+ }
977
+ static #bad(status, scheme, kind, message, extensions = {}) {
978
+ return Results.failure(`scheme:${scheme}`, kind, status, message, { shape: "passthrough" }, extensions);
493
979
  }
494
980
  }
495
981
  _a = Http;