@jini-ai/platform 0.5.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/asset-cache.d.ts +12 -1
- package/dist/asset-cache.js +9 -5
- package/dist/child-environment.d.ts +18 -0
- package/dist/child-environment.js +26 -0
- package/dist/fs/boot-identity.d.ts +10 -0
- package/dist/fs/boot-identity.js +34 -0
- package/dist/fs/directory-home.d.ts +15 -0
- package/dist/fs/directory-home.js +17 -0
- package/dist/fs/directory-registry.d.ts +50 -0
- package/dist/fs/directory-registry.js +99 -0
- package/dist/fs/directory-running-lock.d.ts +43 -0
- package/dist/fs/directory-running-lock.js +66 -0
- package/dist/fs/directory-sync.d.ts +20 -0
- package/dist/fs/directory-sync.js +87 -0
- package/dist/fs/directory-user-config.d.ts +79 -0
- package/dist/fs/directory-user-config.js +183 -0
- package/dist/fs/directory-workspace.d.ts +132 -0
- package/dist/fs/directory-workspace.js +266 -0
- package/dist/fs/durable-json.d.ts +21 -0
- package/dist/fs/durable-json.js +59 -4
- package/dist/fs/file-lock.d.ts +3 -0
- package/dist/fs/file-lock.js +16 -7
- package/dist/fs/index.d.ts +6 -0
- package/dist/fs/index.js +6 -0
- package/dist/fs/strict-containment.d.ts +1 -1
- package/dist/fs/strict-containment.js +1 -1
- package/dist/fs/user-data-directories.d.ts +28 -0
- package/dist/fs/user-data-directories.js +53 -0
- package/dist/fs/verified-directory-move.d.ts +37 -0
- package/dist/fs/verified-directory-move.js +138 -0
- package/dist/fs/verified-file-relocation.d.ts +27 -0
- package/dist/fs/verified-file-relocation.js +115 -0
- package/dist/fs/workspace-directory.d.ts +19 -0
- package/dist/fs/workspace-directory.js +23 -0
- package/dist/http/guarded/transport.fetch.d.ts +13 -1
- package/dist/http/guarded/transport.fetch.js +9 -3
- package/dist/index.d.ts +6 -54
- package/dist/index.js +6 -54
- package/dist/net/address.d.ts +1 -8
- package/dist/net/address.js +2 -89
- package/dist/net/endpoint-policy.d.ts +9 -0
- package/dist/net/endpoint-policy.js +98 -0
- package/dist/secrets/credential-sets/index.d.ts +5 -0
- package/dist/secrets/credential-sets/index.js +5 -0
- package/dist/secrets/credential-sets/repo.memory.d.ts +18 -10
- package/dist/secrets/credential-sets/repo.memory.js +18 -10
- package/dist/secrets/credential-sets/vault.aad.d.ts +4 -0
- package/dist/secrets/credential-sets/vault.aad.js +8 -0
- package/dist/secrets/credential-sets/vault.meta.d.ts +10 -0
- package/dist/secrets/credential-sets/vault.meta.js +52 -0
- package/dist/secrets/credential-sets/vault.repo.memory.d.ts +51 -0
- package/dist/secrets/credential-sets/vault.repo.memory.js +148 -0
- package/dist/secrets/credential-sets/vault.service.d.ts +130 -0
- package/dist/secrets/credential-sets/vault.service.js +310 -0
- package/dist/secrets/credential-sets/vault.types.d.ts +150 -0
- package/dist/secrets/credential-sets/vault.types.js +28 -0
- package/dist/secrets/credential-token.d.ts +64 -0
- package/dist/secrets/credential-token.js +75 -0
- package/dist/secrets/index.d.ts +1 -0
- package/dist/secrets/index.js +1 -0
- package/dist/secrets/secret-sealer.aesgcm.d.ts +6 -0
- package/dist/secrets/secret-sealer.aesgcm.js +9 -2
- package/dist/web/html-document.d.ts +70 -0
- package/dist/web/html-document.js +268 -0
- package/dist/web/html-resource.d.ts +15 -0
- package/dist/web/html-resource.js +12 -0
- package/dist/web/html-to-markdown.d.ts +11 -0
- package/dist/web/html-to-markdown.js +245 -0
- package/dist/web/index.d.ts +5 -0
- package/dist/web/index.js +5 -0
- package/dist/web/web-page.d.ts +157 -0
- package/dist/web/web-page.js +310 -0
- package/package.json +26 -3
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/** Shared credential validation; application detection policy is injected at the metadata boundary. */
|
|
2
|
+
export declare const CREDENTIAL_TOKEN_LIMIT = 8192;
|
|
3
|
+
export declare const CREDENTIAL_MESSAGES: {
|
|
4
|
+
readonly showSecret: "Show";
|
|
5
|
+
readonly hideSecret: "Hide";
|
|
6
|
+
readonly generationUnavailable: "Generation is not available yet for this provider.";
|
|
7
|
+
readonly secretValue: "Secret value";
|
|
8
|
+
readonly saveSecret: "Save secret";
|
|
9
|
+
readonly cancel: "Cancel";
|
|
10
|
+
readonly secretForm: "Enter the value in this secure form.";
|
|
11
|
+
readonly env: "Enter environment variables as NAME=VALUE lines or a JSON object of strings.";
|
|
12
|
+
readonly url: "The server URL must be a valid absolute URL.";
|
|
13
|
+
readonly set: "Access token set";
|
|
14
|
+
readonly storage: "The credential could not be saved or unlocked. Check the site credential store.";
|
|
15
|
+
readonly savedTitle: "Saved.";
|
|
16
|
+
readonly blank: "Enter a token. Spaces alone are not a token.";
|
|
17
|
+
readonly limit: "The token exceeds the 8192-character limit. Copy only the token.";
|
|
18
|
+
readonly ascii: "Hosted tokens must use visible ASCII characters (0x21–0x7E).";
|
|
19
|
+
readonly control: "The token contains an invalid control character.";
|
|
20
|
+
readonly plain: "Put credentials in the secret field, not in this field.";
|
|
21
|
+
readonly saved: "Saved, not tested.";
|
|
22
|
+
readonly connected: "Connected.";
|
|
23
|
+
readonly auth: "The server rejected this token.";
|
|
24
|
+
readonly timeout: "The connection timed out.";
|
|
25
|
+
readonly unreachable: "Could not reach the server.";
|
|
26
|
+
readonly chars: "{length} chars";
|
|
27
|
+
};
|
|
28
|
+
export type CredentialMessageId = keyof typeof CREDENTIAL_MESSAGES;
|
|
29
|
+
/** Errors carry only a rule and field, never the submitted value. */
|
|
30
|
+
export declare class CredentialInputError extends Error {
|
|
31
|
+
readonly rule: CredentialMessageId;
|
|
32
|
+
readonly field: string;
|
|
33
|
+
constructor(rule: CredentialMessageId, field: string);
|
|
34
|
+
}
|
|
35
|
+
/** Validate without normalizing: cards may call this before a probe; only the store trims. */
|
|
36
|
+
export declare function assertCredentialToken({ value, field, hosted }: {
|
|
37
|
+
value: unknown;
|
|
38
|
+
field?: string;
|
|
39
|
+
hosted?: boolean;
|
|
40
|
+
}, _optional?: {}): void;
|
|
41
|
+
/** The single normalization boundary. Interior characters are never rewritten or truncated. */
|
|
42
|
+
export declare function normalizeCredentialToken(required: {
|
|
43
|
+
value: unknown;
|
|
44
|
+
field?: string;
|
|
45
|
+
hosted?: boolean;
|
|
46
|
+
}, _optional?: {}): string;
|
|
47
|
+
export interface CredentialTokenHint {
|
|
48
|
+
length: number;
|
|
49
|
+
last4: string | null;
|
|
50
|
+
}
|
|
51
|
+
/** Call only on a server-side unsealed value; short tokens expose no characters. */
|
|
52
|
+
export declare function credentialTokenHint({ token }: {
|
|
53
|
+
token: string;
|
|
54
|
+
}, _optional?: {}): CredentialTokenHint;
|
|
55
|
+
/** Protect plaintext metadata, using Jini's detector plus structural URL/CLI rules. */
|
|
56
|
+
export declare function assertCredentialFreeField({ value, field, containsSecret }: {
|
|
57
|
+
value: string;
|
|
58
|
+
field: string;
|
|
59
|
+
containsSecret: (required: {
|
|
60
|
+
text: string;
|
|
61
|
+
}) => boolean;
|
|
62
|
+
}, _optional?: {}): void;
|
|
63
|
+
export type CredentialConnection = 'connected' | 'auth' | 'timeout' | 'unreachable' | 'saved';
|
|
64
|
+
//# sourceMappingURL=credential-token.d.ts.map
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/** Shared credential validation; application detection policy is injected at the metadata boundary. */
|
|
2
|
+
export const CREDENTIAL_TOKEN_LIMIT = 8192;
|
|
3
|
+
export const CREDENTIAL_MESSAGES = {
|
|
4
|
+
showSecret: "Show",
|
|
5
|
+
hideSecret: "Hide",
|
|
6
|
+
generationUnavailable: "Generation is not available yet for this provider.",
|
|
7
|
+
secretValue: "Secret value",
|
|
8
|
+
saveSecret: "Save secret",
|
|
9
|
+
cancel: "Cancel",
|
|
10
|
+
secretForm: "Enter the value in this secure form.",
|
|
11
|
+
env: 'Enter environment variables as NAME=VALUE lines or a JSON object of strings.',
|
|
12
|
+
url: 'The server URL must be a valid absolute URL.',
|
|
13
|
+
set: 'Access token set',
|
|
14
|
+
storage: 'The credential could not be saved or unlocked. Check the site credential store.',
|
|
15
|
+
savedTitle: 'Saved.',
|
|
16
|
+
blank: 'Enter a token. Spaces alone are not a token.',
|
|
17
|
+
limit: 'The token exceeds the 8192-character limit. Copy only the token.',
|
|
18
|
+
ascii: 'Hosted tokens must use visible ASCII characters (0x21–0x7E).',
|
|
19
|
+
control: 'The token contains an invalid control character.',
|
|
20
|
+
plain: 'Put credentials in the secret field, not in this field.',
|
|
21
|
+
saved: 'Saved, not tested.',
|
|
22
|
+
connected: 'Connected.',
|
|
23
|
+
auth: 'The server rejected this token.',
|
|
24
|
+
timeout: 'The connection timed out.',
|
|
25
|
+
unreachable: 'Could not reach the server.',
|
|
26
|
+
chars: '{length} chars',
|
|
27
|
+
};
|
|
28
|
+
/** Errors carry only a rule and field, never the submitted value. */
|
|
29
|
+
export class CredentialInputError extends Error {
|
|
30
|
+
rule;
|
|
31
|
+
field;
|
|
32
|
+
constructor(rule, field) {
|
|
33
|
+
super(CREDENTIAL_MESSAGES[rule]);
|
|
34
|
+
this.rule = rule;
|
|
35
|
+
this.field = field;
|
|
36
|
+
this.name = 'CredentialInputError';
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** Validate without normalizing: cards may call this before a probe; only the store trims. */
|
|
40
|
+
export function assertCredentialToken({ value, field = 'token', hosted = true }, _optional = {}) {
|
|
41
|
+
if (typeof value !== 'string' || value.trim() === '')
|
|
42
|
+
throw new CredentialInputError('blank', field);
|
|
43
|
+
// Validate the stored length, so incidental edge whitespace does not consume the token limit.
|
|
44
|
+
const token = value.trim();
|
|
45
|
+
if (token.length > CREDENTIAL_TOKEN_LIMIT)
|
|
46
|
+
throw new CredentialInputError('limit', field);
|
|
47
|
+
if (hosted && /[^\x21-\x7e]/.test(token))
|
|
48
|
+
throw new CredentialInputError('ascii', field);
|
|
49
|
+
if (/[\x00-\x1f\x7f-\x9f]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/u.test(token)) {
|
|
50
|
+
throw new CredentialInputError('control', field);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** The single normalization boundary. Interior characters are never rewritten or truncated. */
|
|
54
|
+
export function normalizeCredentialToken(required, _optional = {}) {
|
|
55
|
+
assertCredentialToken(required);
|
|
56
|
+
return required.value.trim();
|
|
57
|
+
}
|
|
58
|
+
/** Call only on a server-side unsealed value; short tokens expose no characters. */
|
|
59
|
+
export function credentialTokenHint({ token }, _optional = {}) {
|
|
60
|
+
return { length: token.length, last4: token.length >= 12 ? token.slice(-4) : null };
|
|
61
|
+
}
|
|
62
|
+
const SECRET_NAME = /(?:^|[_-])(?:key|api[_-]?key|token|access[_-]?token|secret|password|auth|sig)$/i;
|
|
63
|
+
const SECRET_ARG = /(?:^|\s)--[\w-]*(?:api[_-]?key|token|secret|password)(?:=|\s+)(?!\s*$)\S/i;
|
|
64
|
+
/** Protect plaintext metadata, using Jini's detector plus structural URL/CLI rules. */
|
|
65
|
+
export function assertCredentialFreeField({ value, field, containsSecret }, _optional = {}) {
|
|
66
|
+
let credential = containsSecret({ text: value }) || SECRET_ARG.test(value);
|
|
67
|
+
try {
|
|
68
|
+
const url = new URL(value);
|
|
69
|
+
credential ||= url.username !== '' || url.password !== '' || [...url.searchParams.keys()].some(key => SECRET_NAME.test(key));
|
|
70
|
+
}
|
|
71
|
+
catch { /* A label/argument is ordinarily not an absolute URL. */ }
|
|
72
|
+
if (credential)
|
|
73
|
+
throw new CredentialInputError('plain', field);
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=credential-token.js.map
|
package/dist/secrets/index.d.ts
CHANGED
|
@@ -60,4 +60,5 @@ export type { GeneratedFileRootKey } from "./keyring.env.js";
|
|
|
60
60
|
export { formatAad } from "./aad.js";
|
|
61
61
|
export { defaultPlatformMessages } from "../messages.js";
|
|
62
62
|
export type { PlatformMessages } from "../messages.js";
|
|
63
|
+
export * from "./credential-token.js";
|
|
63
64
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/secrets/index.js
CHANGED
|
@@ -32,4 +32,5 @@ export { deriveFromRootKey } from "./keyring.env.js"; // site-key-legacy: deprec
|
|
|
32
32
|
export { deriveSigningSecretFromRootKey } from "./keyring.env.js"; // site-key-legacy: deprecated alias; remove on/after 2026-11-01
|
|
33
33
|
export { formatAad } from "./aad.js";
|
|
34
34
|
export { defaultPlatformMessages } from "../messages.js";
|
|
35
|
+
export * from "./credential-token.js";
|
|
35
36
|
//# sourceMappingURL=index.js.map
|
|
@@ -2,8 +2,14 @@ import type { KeyringPort, RootKeyHandle, SecretSealerPort } from "./ports.js";
|
|
|
2
2
|
import type { SealedSecret } from "./types.js";
|
|
3
3
|
export declare class AesGcmSecretSealer implements SecretSealerPort {
|
|
4
4
|
private readonly keyring;
|
|
5
|
+
private readonly randomBytesFn;
|
|
6
|
+
/** Inject randomness for wire fixtures; production uses Node's CSPRNG for every fresh IV. */
|
|
5
7
|
constructor({ keyring }: {
|
|
6
8
|
keyring: KeyringPort;
|
|
9
|
+
}, optional?: {
|
|
10
|
+
randomBytesFn?: (input: {
|
|
11
|
+
byteLength: number;
|
|
12
|
+
}) => Uint8Array;
|
|
7
13
|
});
|
|
8
14
|
/** Encrypts under the requested generation with a fresh IV; AAD must be reconstructed on open. */
|
|
9
15
|
seal(input: {
|
|
@@ -14,13 +14,20 @@ export class AesGcmSecretSealer {
|
|
|
14
14
|
// Root keys stay behind handles, and the purpose separates this HKDF domain from signing/tokens.
|
|
15
15
|
// Exercise real GCM with an in-memory keyring: a plaintext fake would miss authentication failures.
|
|
16
16
|
keyring;
|
|
17
|
-
|
|
17
|
+
randomBytesFn;
|
|
18
|
+
/** Inject randomness for wire fixtures; production uses Node's CSPRNG for every fresh IV. */
|
|
19
|
+
constructor({ keyring }, optional = {}) {
|
|
20
|
+
this.keyring = keyring;
|
|
21
|
+
this.randomBytesFn = optional.randomBytesFn ?? (({ byteLength }) => randomBytes(byteLength));
|
|
22
|
+
}
|
|
18
23
|
/** Encrypts under the requested generation with a fresh IV; AAD must be reconstructed on open. */
|
|
19
24
|
async seal(input) {
|
|
20
25
|
if (typeof input.aad !== "string")
|
|
21
26
|
throw new TypeError("AesGcmSecretSealer: aad must be a string");
|
|
22
27
|
const aesKey = await this.deriveAesKey(input.key);
|
|
23
|
-
const iv =
|
|
28
|
+
const iv = Buffer.from(this.randomBytesFn({ byteLength: IV_LENGTH_BYTES }));
|
|
29
|
+
if (iv.length !== IV_LENGTH_BYTES)
|
|
30
|
+
throw new TypeError("AesGcmSecretSealer: IV must contain exactly 12 bytes");
|
|
24
31
|
// GCM must never reuse an IV under the same key. AAD is authenticated but not stored in the
|
|
25
32
|
// envelope, so callers must reconstruct the byte-identical context when opening the record.
|
|
26
33
|
const cipher = createCipheriv(ALG, aesKey, iv);
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { type DefaultTreeAdapterMap } from "parse5";
|
|
2
|
+
type Element = DefaultTreeAdapterMap["element"];
|
|
3
|
+
export interface PageLink {
|
|
4
|
+
href: string;
|
|
5
|
+
text: string;
|
|
6
|
+
internal: boolean;
|
|
7
|
+
}
|
|
8
|
+
export interface PageImage {
|
|
9
|
+
src: string;
|
|
10
|
+
alt: string;
|
|
11
|
+
}
|
|
12
|
+
export interface HtmlDocumentFacts {
|
|
13
|
+
title?: string;
|
|
14
|
+
description?: string;
|
|
15
|
+
lang?: string;
|
|
16
|
+
links: PageLink[];
|
|
17
|
+
linksTruncated: boolean;
|
|
18
|
+
images: PageImage[];
|
|
19
|
+
imagesTruncated: boolean;
|
|
20
|
+
stylesheets: string[];
|
|
21
|
+
meta: Record<string, string>;
|
|
22
|
+
}
|
|
23
|
+
export declare const MAX_LINKS = 500;
|
|
24
|
+
export declare const MAX_IMAGES = 300;
|
|
25
|
+
export declare const LINK_PROTOCOLS: ReadonlySet<string>;
|
|
26
|
+
export declare const ASSET_PROTOCOLS: ReadonlySet<string>;
|
|
27
|
+
export declare function attr({ element, name }: {
|
|
28
|
+
element: Element;
|
|
29
|
+
name: string;
|
|
30
|
+
}, _optional?: Record<string, never>): string | undefined;
|
|
31
|
+
/** Resolves against `base`; drops fragment-only, unparseable and other-protocol values, and fragments. */
|
|
32
|
+
export declare function absolute({ raw, base, protocols }: {
|
|
33
|
+
raw: string | undefined;
|
|
34
|
+
base: string;
|
|
35
|
+
protocols: ReadonlySet<string>;
|
|
36
|
+
}, _optional?: Record<string, never>): string | undefined;
|
|
37
|
+
/** First usable image URL: lazy loaders park the real one in `data-src` behind a `data:` placeholder. */
|
|
38
|
+
export declare function imageSource({ img }: {
|
|
39
|
+
img: Element;
|
|
40
|
+
}, _optional?: Record<string, never>): string | undefined;
|
|
41
|
+
/**
|
|
42
|
+
* Extracts page facts. Relative URLs resolve against `<base href>` (itself resolved against the page
|
|
43
|
+
* URL) so links match what a browser would follow.
|
|
44
|
+
* @param required.html - Decoded page source; untrusted.
|
|
45
|
+
* @param required.pageUrl - The final URL the HTML was served from (after redirects).
|
|
46
|
+
* @returns Facts with every URL absolute; links deduped (fragment-free) and capped at {@link MAX_LINKS},
|
|
47
|
+
* images at {@link MAX_IMAGES}; `internal` ignores a leading `www.`.
|
|
48
|
+
* @complexity O(nodes) plus O(anchor subtree) per anchor for its text.
|
|
49
|
+
*/
|
|
50
|
+
export declare function readHtmlDocument({ html, pageUrl }: {
|
|
51
|
+
html: string;
|
|
52
|
+
pageUrl: string;
|
|
53
|
+
}, _optional?: Record<string, never>): HtmlDocumentFacts;
|
|
54
|
+
/**
|
|
55
|
+
* Visible text with one line per block element; scripts, styles, `<head>`, SVG and embeds omitted.
|
|
56
|
+
* @complexity O(nodes).
|
|
57
|
+
*/
|
|
58
|
+
export declare function htmlToText({ html }: {
|
|
59
|
+
html: string;
|
|
60
|
+
}, _optional?: Record<string, never>): string;
|
|
61
|
+
/**
|
|
62
|
+
* The page HTML with `<script>`, `<noscript>`, `<template>`, comments, `on*` handler attributes and
|
|
63
|
+
* `javascript:` attribute values removed (styles and classes kept: theming needs them). Re-serialized by parse5, so it is well-formed rather than byte-identical.
|
|
64
|
+
* @complexity O(nodes).
|
|
65
|
+
*/
|
|
66
|
+
export declare function stripActiveHtml({ html }: {
|
|
67
|
+
html: string;
|
|
68
|
+
}, _optional?: Record<string, never>): string;
|
|
69
|
+
export {};
|
|
70
|
+
//# sourceMappingURL=html-document.d.ts.map
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
import { parse, serialize } from "parse5";
|
|
2
|
+
export const MAX_LINKS = 500;
|
|
3
|
+
export const MAX_IMAGES = 300;
|
|
4
|
+
const MAX_STYLESHEETS = 100;
|
|
5
|
+
const MAX_META = 60;
|
|
6
|
+
const MAX_LINK_TEXT = 200;
|
|
7
|
+
const MAX_META_VALUE = 500;
|
|
8
|
+
export const LINK_PROTOCOLS = new Set(["http:", "https:", "mailto:", "tel:"]);
|
|
9
|
+
export const ASSET_PROTOCOLS = new Set(["http:", "https:"]);
|
|
10
|
+
/** Never visible and never page content; removed from the text and the returned HTML copy. */
|
|
11
|
+
const STRIPPED = new Set(["script", "noscript", "template"]);
|
|
12
|
+
/** Not visible text either, but `style` stays in the HTML copy because theming needs it. */
|
|
13
|
+
const INVISIBLE = new Set([...STRIPPED, "style", "head", "svg", "iframe", "object", "canvas"]);
|
|
14
|
+
const BLOCKS = new Set(["address", "article", "aside", "blockquote", "br", "dd", "div", "dl", "dt", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "ol", "p", "pre", "section", "table", "td", "th", "tr", "ul"]);
|
|
15
|
+
/** `meta name=` keys worth returning; every `og:*` property is returned as well. */
|
|
16
|
+
const META_NAMES = new Set(["generator", "theme-color", "author", "keywords", "twitter:title", "twitter:description", "twitter:image", "application-name"]);
|
|
17
|
+
export function attr({ element, name }, _optional = {}) {
|
|
18
|
+
return element.attrs.find(a => a.name === name)?.value;
|
|
19
|
+
}
|
|
20
|
+
function isElement(node) {
|
|
21
|
+
return "tagName" in node;
|
|
22
|
+
}
|
|
23
|
+
/** Iterative pre-order walk: deeply nested hostile HTML cannot overflow the stack. O(nodes). */
|
|
24
|
+
function* walk(root) {
|
|
25
|
+
const pending = [root];
|
|
26
|
+
while (pending.length) {
|
|
27
|
+
const node = pending.pop();
|
|
28
|
+
yield node;
|
|
29
|
+
// Read after the yield on purpose: stripActiveHtml filters childNodes in between.
|
|
30
|
+
const children = "childNodes" in node ? node.childNodes : [];
|
|
31
|
+
for (let i = children.length - 1; i >= 0; i--)
|
|
32
|
+
pending.push(children[i]);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
function collapse(text) {
|
|
36
|
+
return text.replace(/\s+/g, " ").trim();
|
|
37
|
+
}
|
|
38
|
+
function textOf(root) {
|
|
39
|
+
const parts = [];
|
|
40
|
+
for (const node of walk(root))
|
|
41
|
+
if (node.nodeName === "#text" && "value" in node)
|
|
42
|
+
parts.push(node.value);
|
|
43
|
+
return collapse(parts.join(" "));
|
|
44
|
+
}
|
|
45
|
+
/** Resolves against `base`; drops fragment-only, unparseable and other-protocol values, and fragments. */
|
|
46
|
+
export function absolute({ raw, base, protocols }, _optional = {}) {
|
|
47
|
+
const value = raw?.trim();
|
|
48
|
+
if (!value || value.startsWith("#"))
|
|
49
|
+
return undefined;
|
|
50
|
+
let url;
|
|
51
|
+
try {
|
|
52
|
+
url = new URL(value, base);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return undefined;
|
|
56
|
+
}
|
|
57
|
+
if (!protocols.has(url.protocol))
|
|
58
|
+
return undefined;
|
|
59
|
+
if (url.protocol === "http:" || url.protocol === "https:")
|
|
60
|
+
url.hash = "";
|
|
61
|
+
return url.href;
|
|
62
|
+
}
|
|
63
|
+
function siteHost(hostname) {
|
|
64
|
+
return hostname.toLowerCase().replace(/^www\./, "");
|
|
65
|
+
}
|
|
66
|
+
function isInternal(href, pageHost) {
|
|
67
|
+
const url = new URL(href);
|
|
68
|
+
return (url.protocol === "http:" || url.protocol === "https:") && siteHost(url.hostname) === pageHost;
|
|
69
|
+
}
|
|
70
|
+
function linkText(anchor) {
|
|
71
|
+
const text = textOf(anchor) || attr({ element: anchor, name: "aria-label" }) || attr({ element: anchor, name: "title" }) || "";
|
|
72
|
+
if (text)
|
|
73
|
+
return collapse(text).slice(0, MAX_LINK_TEXT);
|
|
74
|
+
for (const node of walk(anchor))
|
|
75
|
+
if (isElement(node) && node.tagName === "img")
|
|
76
|
+
return collapse(attr({ element: node, name: "alt" }) ?? "").slice(0, MAX_LINK_TEXT);
|
|
77
|
+
return "";
|
|
78
|
+
}
|
|
79
|
+
/** First usable image URL: lazy loaders park the real one in `data-src` behind a `data:` placeholder. */
|
|
80
|
+
export function imageSource({ img }, _optional = {}) {
|
|
81
|
+
const src = attr({ element: img, name: "src" });
|
|
82
|
+
if (src && !src.trim().startsWith("data:"))
|
|
83
|
+
return src;
|
|
84
|
+
return attr({ element: img, name: "data-src" }) ?? attr({ element: img, name: "data-lazy-src" }) ?? attr({ element: img, name: "srcset" })?.trim().split(/\s+/)[0];
|
|
85
|
+
}
|
|
86
|
+
function collect(document) {
|
|
87
|
+
const raw = { anchors: [], images: [], linkTags: [], metas: [] };
|
|
88
|
+
for (const node of walk(document)) {
|
|
89
|
+
if (!isElement(node))
|
|
90
|
+
continue;
|
|
91
|
+
switch (node.tagName) {
|
|
92
|
+
case "a":
|
|
93
|
+
raw.anchors.push(node);
|
|
94
|
+
break;
|
|
95
|
+
case "img":
|
|
96
|
+
raw.images.push(node);
|
|
97
|
+
break;
|
|
98
|
+
case "link":
|
|
99
|
+
raw.linkTags.push(node);
|
|
100
|
+
break;
|
|
101
|
+
case "meta":
|
|
102
|
+
raw.metas.push(node);
|
|
103
|
+
break;
|
|
104
|
+
case "title":
|
|
105
|
+
raw.title ??= node;
|
|
106
|
+
break;
|
|
107
|
+
case "html":
|
|
108
|
+
raw.html ??= node;
|
|
109
|
+
break;
|
|
110
|
+
case "base":
|
|
111
|
+
raw.base ??= attr({ element: node, name: "href" });
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return raw;
|
|
116
|
+
}
|
|
117
|
+
function readMeta(raw, base) {
|
|
118
|
+
const meta = {};
|
|
119
|
+
const put = (key, value) => {
|
|
120
|
+
if (value === undefined || key in meta || Object.keys(meta).length >= MAX_META)
|
|
121
|
+
return;
|
|
122
|
+
meta[key] = collapse(value).slice(0, MAX_META_VALUE);
|
|
123
|
+
};
|
|
124
|
+
for (const element of raw.metas) {
|
|
125
|
+
const property = attr({ element, name: "property" })?.toLowerCase();
|
|
126
|
+
const name = attr({ element, name: "name" })?.toLowerCase();
|
|
127
|
+
const content = attr({ element, name: "content" });
|
|
128
|
+
if (property?.startsWith("og:"))
|
|
129
|
+
put(property, property === "og:image" || property === "og:url" ? absolute({ raw: content, base, protocols: ASSET_PROTOCOLS }) : content);
|
|
130
|
+
else if (name && META_NAMES.has(name))
|
|
131
|
+
put(name, name === "twitter:image" ? absolute({ raw: content, base, protocols: ASSET_PROTOCOLS }) : content);
|
|
132
|
+
}
|
|
133
|
+
for (const element of raw.linkTags) {
|
|
134
|
+
const rel = (attr({ element, name: "rel" }) ?? "").toLowerCase().split(/\s+/);
|
|
135
|
+
const href = absolute({ raw: attr({ element, name: "href" }), base, protocols: ASSET_PROTOCOLS });
|
|
136
|
+
if (rel.includes("canonical"))
|
|
137
|
+
put("canonical", href);
|
|
138
|
+
if (rel.includes("icon"))
|
|
139
|
+
put("favicon", href);
|
|
140
|
+
if (rel.includes("apple-touch-icon"))
|
|
141
|
+
put("apple-touch-icon", href);
|
|
142
|
+
}
|
|
143
|
+
return meta;
|
|
144
|
+
}
|
|
145
|
+
function readLinks(anchors, base, pageHost) {
|
|
146
|
+
const seen = new Set();
|
|
147
|
+
const links = [];
|
|
148
|
+
let truncated = false;
|
|
149
|
+
for (const anchor of anchors) {
|
|
150
|
+
const href = absolute({ raw: attr({ element: anchor, name: "href" }), base, protocols: LINK_PROTOCOLS });
|
|
151
|
+
if (!href || seen.has(href))
|
|
152
|
+
continue;
|
|
153
|
+
seen.add(href);
|
|
154
|
+
if (links.length === MAX_LINKS) {
|
|
155
|
+
truncated = true;
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
links.push({ href, text: linkText(anchor), internal: isInternal(href, pageHost) });
|
|
159
|
+
}
|
|
160
|
+
return { links, truncated };
|
|
161
|
+
}
|
|
162
|
+
function readImages(images, base) {
|
|
163
|
+
const seen = new Set();
|
|
164
|
+
const out = [];
|
|
165
|
+
let truncated = false;
|
|
166
|
+
for (const img of images) {
|
|
167
|
+
const src = absolute({ raw: imageSource({ img }), base, protocols: ASSET_PROTOCOLS });
|
|
168
|
+
if (!src || seen.has(src))
|
|
169
|
+
continue;
|
|
170
|
+
seen.add(src);
|
|
171
|
+
if (out.length === MAX_IMAGES) {
|
|
172
|
+
truncated = true;
|
|
173
|
+
break;
|
|
174
|
+
}
|
|
175
|
+
out.push({ src, alt: collapse(attr({ element: img, name: "alt" }) ?? "") });
|
|
176
|
+
}
|
|
177
|
+
return { images: out, truncated };
|
|
178
|
+
}
|
|
179
|
+
function readStylesheets(linkTags, base) {
|
|
180
|
+
const out = new Set();
|
|
181
|
+
for (const element of linkTags) {
|
|
182
|
+
const rel = (attr({ element, name: "rel" }) ?? "").toLowerCase().split(/\s+/);
|
|
183
|
+
const href = absolute({ raw: attr({ element, name: "href" }), base, protocols: ASSET_PROTOCOLS });
|
|
184
|
+
if (rel.includes("stylesheet") && href && out.size < MAX_STYLESHEETS)
|
|
185
|
+
out.add(href);
|
|
186
|
+
}
|
|
187
|
+
return [...out];
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Extracts page facts. Relative URLs resolve against `<base href>` (itself resolved against the page
|
|
191
|
+
* URL) so links match what a browser would follow.
|
|
192
|
+
* @param required.html - Decoded page source; untrusted.
|
|
193
|
+
* @param required.pageUrl - The final URL the HTML was served from (after redirects).
|
|
194
|
+
* @returns Facts with every URL absolute; links deduped (fragment-free) and capped at {@link MAX_LINKS},
|
|
195
|
+
* images at {@link MAX_IMAGES}; `internal` ignores a leading `www.`.
|
|
196
|
+
* @complexity O(nodes) plus O(anchor subtree) per anchor for its text.
|
|
197
|
+
*/
|
|
198
|
+
export function readHtmlDocument({ html, pageUrl }, _optional = {}) {
|
|
199
|
+
const document = parse(html);
|
|
200
|
+
const raw = collect(document);
|
|
201
|
+
const base = absolute({ raw: raw.base, base: pageUrl, protocols: ASSET_PROTOCOLS }) ?? pageUrl;
|
|
202
|
+
const pageHost = siteHost(new URL(pageUrl).hostname);
|
|
203
|
+
const links = readLinks(raw.anchors, base, pageHost);
|
|
204
|
+
const images = readImages(raw.images, base);
|
|
205
|
+
const description = raw.metas.find(m => attr({ element: m, name: "name" })?.toLowerCase() === "description");
|
|
206
|
+
const title = raw.title ? textOf(raw.title) : "";
|
|
207
|
+
const lang = raw.html ? attr({ element: raw.html, name: "lang" })?.trim() : undefined;
|
|
208
|
+
const descriptionText = description ? collapse(attr({ element: description, name: "content" }) ?? "") : "";
|
|
209
|
+
return {
|
|
210
|
+
...(title ? { title } : {}),
|
|
211
|
+
...(descriptionText ? { description: descriptionText } : {}),
|
|
212
|
+
...(lang ? { lang } : {}),
|
|
213
|
+
links: links.links, linksTruncated: links.truncated,
|
|
214
|
+
images: images.images, imagesTruncated: images.truncated,
|
|
215
|
+
stylesheets: readStylesheets(raw.linkTags, base),
|
|
216
|
+
meta: readMeta(raw, base),
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Visible text with one line per block element; scripts, styles, `<head>`, SVG and embeds omitted.
|
|
221
|
+
* @complexity O(nodes).
|
|
222
|
+
*/
|
|
223
|
+
export function htmlToText({ html }, _optional = {}) {
|
|
224
|
+
const pending = [parse(html)];
|
|
225
|
+
const out = [];
|
|
226
|
+
while (pending.length) {
|
|
227
|
+
const node = pending.pop();
|
|
228
|
+
if (node === "\n") {
|
|
229
|
+
out.push("\n");
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
if (isElement(node) && INVISIBLE.has(node.tagName))
|
|
233
|
+
continue;
|
|
234
|
+
if (node.nodeName === "#text" && "value" in node) {
|
|
235
|
+
out.push(node.value);
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
if (!("childNodes" in node))
|
|
239
|
+
continue;
|
|
240
|
+
const block = isElement(node) && BLOCKS.has(node.tagName);
|
|
241
|
+
if (block) {
|
|
242
|
+
out.push("\n");
|
|
243
|
+
pending.push("\n");
|
|
244
|
+
}
|
|
245
|
+
for (let i = node.childNodes.length - 1; i >= 0; i--)
|
|
246
|
+
pending.push(node.childNodes[i]);
|
|
247
|
+
}
|
|
248
|
+
return out.join("").split("\n").map(line => line.replace(/[^\S\n]+/g, " ").trim()).filter(Boolean).join("\n");
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* The page HTML with `<script>`, `<noscript>`, `<template>`, comments, `on*` handler attributes and
|
|
252
|
+
* `javascript:` attribute values removed (styles and classes kept: theming needs them). Re-serialized by parse5, so it is well-formed rather than byte-identical.
|
|
253
|
+
* @complexity O(nodes).
|
|
254
|
+
*/
|
|
255
|
+
export function stripActiveHtml({ html }, _optional = {}) {
|
|
256
|
+
const document = parse(html);
|
|
257
|
+
for (const node of walk(document)) {
|
|
258
|
+
if (!("childNodes" in node))
|
|
259
|
+
continue;
|
|
260
|
+
const parent = node;
|
|
261
|
+
parent.childNodes = parent.childNodes.filter(child => child.nodeName !== "#comment" && !(isElement(child) && STRIPPED.has(child.tagName)));
|
|
262
|
+
// Inert copy: an agent may paste this into a page, so handlers and script URLs never survive.
|
|
263
|
+
if (isElement(node))
|
|
264
|
+
node.attrs = node.attrs.filter(a => !/^on/i.test(a.name) && !/^\s*javascript:/i.test(a.value));
|
|
265
|
+
}
|
|
266
|
+
return serialize(document);
|
|
267
|
+
}
|
|
268
|
+
//# sourceMappingURL=html-document.js.map
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Bound synchronous HTML parsers independently of the HTTP response byte limit. */
|
|
2
|
+
export declare const MAX_HTML_CHARS = 1000000;
|
|
3
|
+
/** Return a bounded prefix without splitting UTF-16; fetch projections expose its truncation flag.
|
|
4
|
+
* The caller may supply a lower projection budget; direct conversion always has a fixed ceiling.
|
|
5
|
+
* O(1) decision plus O(cap) prefix allocation; raw body output keeps its separate transport cap.
|
|
6
|
+
*/
|
|
7
|
+
export declare function boundHtml({ html }: {
|
|
8
|
+
html: string;
|
|
9
|
+
}, { maxChars }?: {
|
|
10
|
+
maxChars?: number;
|
|
11
|
+
}): {
|
|
12
|
+
html: string;
|
|
13
|
+
truncated: boolean;
|
|
14
|
+
};
|
|
15
|
+
//# sourceMappingURL=html-resource.d.ts.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Bound synchronous HTML parsers independently of the HTTP response byte limit. */
|
|
2
|
+
export const MAX_HTML_CHARS = 1_000_000;
|
|
3
|
+
/** Return a bounded prefix without splitting UTF-16; fetch projections expose its truncation flag.
|
|
4
|
+
* The caller may supply a lower projection budget; direct conversion always has a fixed ceiling.
|
|
5
|
+
* O(1) decision plus O(cap) prefix allocation; raw body output keeps its separate transport cap.
|
|
6
|
+
*/
|
|
7
|
+
export function boundHtml({ html }, { maxChars = MAX_HTML_CHARS } = {}) {
|
|
8
|
+
const limit = Math.min(MAX_HTML_CHARS, maxChars);
|
|
9
|
+
const end = /[\uDC00-\uDFFF]/.test(html[limit] ?? "") ? limit - 1 : limit;
|
|
10
|
+
return html.length > limit ? { html: html.slice(0, end), truncated: true } : { html, truncated: false };
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=html-resource.js.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert untrusted HTML to readable Markdown with absolute, allowlisted link/image URLs.
|
|
3
|
+
* @returns Markdown without surrounding whitespace; input is capped before parsing and errors propagate.
|
|
4
|
+
* @complexity O(nodes + output size) time/space for ordinary pages; nesting is bounded at 300.
|
|
5
|
+
* @example htmlToMarkdown({ html: "<h1>Hello</h1>", pageUrl: "https://example.com/" })
|
|
6
|
+
*/
|
|
7
|
+
export declare function htmlToMarkdown({ html, pageUrl }: {
|
|
8
|
+
html: string;
|
|
9
|
+
pageUrl: string;
|
|
10
|
+
}, _optional?: Record<string, never>): string;
|
|
11
|
+
//# sourceMappingURL=html-to-markdown.d.ts.map
|