@jini-ai/platform 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/README.md +1 -1
- package/dist/asset-cache.d.ts +12 -1
- package/dist/asset-cache.js +9 -5
- package/dist/child-environment.d.ts +18 -0
- package/dist/child-environment.js +26 -0
- package/dist/fs/boot-identity.d.ts +10 -0
- package/dist/fs/boot-identity.js +34 -0
- package/dist/fs/directory-home.d.ts +15 -0
- package/dist/fs/directory-home.js +17 -0
- package/dist/fs/directory-registry.d.ts +50 -0
- package/dist/fs/directory-registry.js +99 -0
- package/dist/fs/directory-running-lock.d.ts +43 -0
- package/dist/fs/directory-running-lock.js +66 -0
- package/dist/fs/directory-sync.d.ts +20 -0
- package/dist/fs/directory-sync.js +87 -0
- package/dist/fs/directory-user-config.d.ts +79 -0
- package/dist/fs/directory-user-config.js +183 -0
- package/dist/fs/directory-workspace.d.ts +132 -0
- package/dist/fs/directory-workspace.js +266 -0
- package/dist/fs/durable-json.d.ts +21 -0
- package/dist/fs/durable-json.js +59 -4
- package/dist/fs/file-lock.d.ts +3 -0
- package/dist/fs/file-lock.js +16 -7
- package/dist/fs/index.d.ts +6 -0
- package/dist/fs/index.js +6 -0
- package/dist/fs/user-data-directories.d.ts +28 -0
- package/dist/fs/user-data-directories.js +53 -0
- package/dist/fs/verified-directory-move.d.ts +37 -0
- package/dist/fs/verified-directory-move.js +138 -0
- package/dist/fs/verified-file-relocation.d.ts +27 -0
- package/dist/fs/verified-file-relocation.js +115 -0
- package/dist/fs/workspace-directory.d.ts +19 -0
- package/dist/fs/workspace-directory.js +23 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/secrets/credential-sets/index.d.ts +5 -2
- package/dist/secrets/credential-sets/index.js +5 -2
- package/dist/secrets/credential-sets/vault.aad.d.ts +4 -0
- package/dist/secrets/credential-sets/vault.aad.js +8 -0
- package/dist/secrets/credential-sets/vault.meta.d.ts +10 -0
- package/dist/secrets/credential-sets/vault.meta.js +52 -0
- package/dist/secrets/credential-sets/vault.repo.memory.d.ts +51 -0
- package/dist/secrets/credential-sets/vault.repo.memory.js +148 -0
- package/dist/secrets/credential-sets/vault.service.d.ts +115 -0
- package/dist/secrets/credential-sets/vault.service.js +273 -0
- package/dist/secrets/credential-sets/vault.types.d.ts +151 -0
- package/dist/secrets/credential-sets/vault.types.js +28 -0
- package/dist/secrets/testing.d.ts +0 -1
- package/dist/secrets/testing.js +0 -1
- package/dist/web/html-document.d.ts +70 -0
- package/dist/web/html-document.js +268 -0
- package/dist/web/html-resource.d.ts +15 -0
- package/dist/web/html-resource.js +12 -0
- package/dist/web/html-to-markdown.d.ts +11 -0
- package/dist/web/html-to-markdown.js +245 -0
- package/dist/web/index.d.ts +5 -0
- package/dist/web/index.js +5 -0
- package/dist/web/web-page.d.ts +157 -0
- package/dist/web/web-page.js +310 -0
- package/package.json +14 -3
- package/dist/secrets/credential-sets/aad.d.ts +0 -7
- package/dist/secrets/credential-sets/aad.js +0 -7
- package/dist/secrets/credential-sets/repo.memory.d.ts +0 -44
- package/dist/secrets/credential-sets/repo.memory.js +0 -80
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import type { SealedSecret } from "../types.js";
|
|
2
|
+
export declare const VAULT_KINDS: readonly ["site-assistant", "personal-execution", "media-provider", "custom", "publish-peer", "database-destination", "vendor", "source-control", "mcp-env", "mcp-auth"];
|
|
3
|
+
export type VaultKind = typeof VAULT_KINDS[number];
|
|
4
|
+
export type VaultJson = null | boolean | number | string | VaultJson[] | {
|
|
5
|
+
[key: string]: VaultJson;
|
|
6
|
+
};
|
|
7
|
+
export type VaultMeta = {
|
|
8
|
+
[key: string]: VaultJson;
|
|
9
|
+
};
|
|
10
|
+
export type VaultOptions = Record<string, never>;
|
|
11
|
+
/** Expected ownership is supplied by the caller, never reconstructed from an untrusted row. */
|
|
12
|
+
export interface VaultContext {
|
|
13
|
+
workspaceId: string;
|
|
14
|
+
kind: VaultKind;
|
|
15
|
+
ownerKey: string;
|
|
16
|
+
principalId: string | null;
|
|
17
|
+
/** Host-canonicalized endpoint/resource binding; null for credentials without such a binding. */
|
|
18
|
+
binding: string | null;
|
|
19
|
+
}
|
|
20
|
+
export interface VaultIdentity extends VaultContext {
|
|
21
|
+
id: string;
|
|
22
|
+
}
|
|
23
|
+
export interface VaultRecord extends VaultIdentity {
|
|
24
|
+
provider: string | null;
|
|
25
|
+
label: string;
|
|
26
|
+
isDefault: boolean;
|
|
27
|
+
revision: number;
|
|
28
|
+
sealed: SealedSecret | null;
|
|
29
|
+
last4: string | null;
|
|
30
|
+
tokenLength: number | null;
|
|
31
|
+
meta: VaultMeta;
|
|
32
|
+
createdAt: string;
|
|
33
|
+
updatedAt: string;
|
|
34
|
+
}
|
|
35
|
+
export type VaultSummary = Omit<VaultRecord, "sealed"> & {
|
|
36
|
+
configured: boolean;
|
|
37
|
+
};
|
|
38
|
+
export interface VaultQuery {
|
|
39
|
+
workspaceId: string;
|
|
40
|
+
kind: VaultKind;
|
|
41
|
+
ownerKey: string;
|
|
42
|
+
principalId: string | null;
|
|
43
|
+
provider?: string | null;
|
|
44
|
+
}
|
|
45
|
+
/** A default belongs to one exact group; omitted provider never means all providers. */
|
|
46
|
+
export interface VaultDefaultQuery extends VaultQuery {
|
|
47
|
+
provider: string | null;
|
|
48
|
+
}
|
|
49
|
+
/** Singleton lookup ignores provider/default; omitted owner/principal leave that dimension unfiltered.
|
|
50
|
+
* Hosts enforce singleton cardinality for their kind/scope. No binding is inferred by the service.
|
|
51
|
+
*/
|
|
52
|
+
export interface VaultSingletonQuery {
|
|
53
|
+
workspaceId: string;
|
|
54
|
+
kind: VaultKind;
|
|
55
|
+
ownerKey?: string | undefined;
|
|
56
|
+
principalId?: string | null | undefined;
|
|
57
|
+
}
|
|
58
|
+
/** Ordered CAS operations; default side effects affect later operations' revisions. */
|
|
59
|
+
export type VaultBatchOp = {
|
|
60
|
+
put: {
|
|
61
|
+
record: VaultRecord;
|
|
62
|
+
expectedRevision: number | null;
|
|
63
|
+
};
|
|
64
|
+
} | {
|
|
65
|
+
delete: {
|
|
66
|
+
workspaceId: string;
|
|
67
|
+
id: string;
|
|
68
|
+
expectedRevision: number;
|
|
69
|
+
};
|
|
70
|
+
} | {
|
|
71
|
+
patchMeta: {
|
|
72
|
+
workspaceId: string;
|
|
73
|
+
id: string;
|
|
74
|
+
expectedRevision: number;
|
|
75
|
+
patch: VaultMeta;
|
|
76
|
+
updatedAt: string;
|
|
77
|
+
};
|
|
78
|
+
};
|
|
79
|
+
export type VaultBatchResult = VaultRecord | boolean;
|
|
80
|
+
export interface VaultCopyResult {
|
|
81
|
+
kind: VaultKind;
|
|
82
|
+
id: string;
|
|
83
|
+
status: "copied" | "skipped" | "failed";
|
|
84
|
+
last4?: string;
|
|
85
|
+
reasonCode?: VaultError["code"] | "source-needs-upgrade" | "source-missing" | "target-exists" | "kind-not-copyable" | "requires-mcp-configuration" | "copy-failed";
|
|
86
|
+
}
|
|
87
|
+
export type VaultMetaParser = (input: {
|
|
88
|
+
meta: unknown;
|
|
89
|
+
}, options?: VaultOptions) => VaultMeta;
|
|
90
|
+
export type VaultMetaParsers = Readonly<Record<VaultKind, VaultMetaParser>>;
|
|
91
|
+
/** Safe error codes never contain submitted values or encrypted material. */
|
|
92
|
+
export declare class VaultError extends Error {
|
|
93
|
+
readonly code: "conflict" | "context" | "invalid-meta" | "invalid-record";
|
|
94
|
+
constructor({ code }: {
|
|
95
|
+
code: VaultError["code"];
|
|
96
|
+
}, _options?: VaultOptions);
|
|
97
|
+
}
|
|
98
|
+
/** CAS is mandatory: null creates only; a number replaces/deletes only that revision.
|
|
99
|
+
* Defaults group by workspace/kind/owner/principal/provider. Every default change increments
|
|
100
|
+
* revision; promotion uses updatedAt descending, then id descending (binary UTF-8 order).
|
|
101
|
+
* All writes must join a host transaction when the adapter supports transaction composition.
|
|
102
|
+
*/
|
|
103
|
+
export interface VaultRepoPort {
|
|
104
|
+
find(input: {
|
|
105
|
+
workspaceId: string;
|
|
106
|
+
id: string;
|
|
107
|
+
}, options?: VaultOptions): Promise<VaultRecord | null>;
|
|
108
|
+
/** Indexed exact-group lookup; returns at most one row without listing its siblings.
|
|
109
|
+
* Optional for compatibility with pre-C1 ports; both built-in adapters implement it.
|
|
110
|
+
*/
|
|
111
|
+
findDefault?(input: VaultDefaultQuery, options?: VaultOptions): Promise<VaultRecord | null>;
|
|
112
|
+
/** Optional additions keep older injected ports source-compatible; built-in adapters implement both. */
|
|
113
|
+
findOne?(input: VaultSingletonQuery, options?: VaultOptions): Promise<VaultRecord | null>;
|
|
114
|
+
/** All operations commit together or throw with no published changes. Default side effects
|
|
115
|
+
* follow operation order; a delete carries full expected ownership.
|
|
116
|
+
* SQL joins the host transaction: callers must propagate failures to roll that transaction back.
|
|
117
|
+
* @example await repo.applyBatch!({ ops: [{ put: { record, expectedRevision: null } }] });
|
|
118
|
+
*/
|
|
119
|
+
applyBatch?(input: {
|
|
120
|
+
ops: readonly VaultBatchOp[];
|
|
121
|
+
}, options?: VaultOptions): Promise<VaultBatchResult[]>;
|
|
122
|
+
list(input: VaultQuery, options?: VaultOptions): Promise<VaultRecord[]>;
|
|
123
|
+
put(input: {
|
|
124
|
+
record: VaultRecord;
|
|
125
|
+
expectedRevision: number | null;
|
|
126
|
+
}, options?: VaultOptions): Promise<VaultRecord>;
|
|
127
|
+
/** Service-validated patch: top-level keys are plain JSON-path identifiers. */
|
|
128
|
+
patchMeta(input: {
|
|
129
|
+
workspaceId: string;
|
|
130
|
+
id: string;
|
|
131
|
+
expectedRevision: number;
|
|
132
|
+
patch: VaultMeta;
|
|
133
|
+
updatedAt: string;
|
|
134
|
+
}, options?: VaultOptions): Promise<VaultRecord>;
|
|
135
|
+
delete(input: {
|
|
136
|
+
workspaceId: string;
|
|
137
|
+
id: string;
|
|
138
|
+
expectedRevision: number;
|
|
139
|
+
}, options?: VaultOptions): Promise<boolean>;
|
|
140
|
+
}
|
|
141
|
+
/** Unambiguous scope encoding keeps adapter default/identity checks on the same contract. */
|
|
142
|
+
export declare function vaultGroupKey({ record }: {
|
|
143
|
+
record: VaultRecord;
|
|
144
|
+
}, _options?: VaultOptions): string;
|
|
145
|
+
/** Adapters guard CAS and immutable identity even for callers below the service. */
|
|
146
|
+
export declare function assertVaultReplacement({ record, existing, expectedRevision }: {
|
|
147
|
+
record: VaultRecord;
|
|
148
|
+
existing: VaultRecord | null;
|
|
149
|
+
expectedRevision: number | null;
|
|
150
|
+
}, _options?: VaultOptions): void;
|
|
151
|
+
//# sourceMappingURL=vault.types.d.ts.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export const VAULT_KINDS = ["site-assistant", "personal-execution", "media-provider", "custom", "publish-peer",
|
|
2
|
+
"database-destination", "vendor", "source-control", "mcp-env", "mcp-auth"];
|
|
3
|
+
/** Safe error codes never contain submitted values or encrypted material. */
|
|
4
|
+
export class VaultError extends Error {
|
|
5
|
+
code;
|
|
6
|
+
constructor({ code }, _options = {}) {
|
|
7
|
+
super(`Secrets vault: ${code}`);
|
|
8
|
+
this.code = code;
|
|
9
|
+
this.name = "VaultError";
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
/** Unambiguous scope encoding keeps adapter default/identity checks on the same contract. */
|
|
13
|
+
export function vaultGroupKey({ record }, _options = {}) {
|
|
14
|
+
return JSON.stringify([record.workspaceId, record.kind, record.ownerKey, record.principalId, record.provider]);
|
|
15
|
+
}
|
|
16
|
+
/** Adapters guard CAS and immutable identity even for callers below the service. */
|
|
17
|
+
export function assertVaultReplacement({ record, existing, expectedRevision }, _options = {}) {
|
|
18
|
+
if (expectedRevision === null ? existing !== null : existing?.revision !== expectedRevision)
|
|
19
|
+
throw new VaultError({ code: "conflict" });
|
|
20
|
+
if (!Number.isSafeInteger(record.revision) || record.revision !== (expectedRevision ?? 0) + 1)
|
|
21
|
+
throw new VaultError({ code: "invalid-record" });
|
|
22
|
+
// Provider groups defaults, but is not authenticated identity (see buildVaultAad).
|
|
23
|
+
if (existing && (record.workspaceId !== existing.workspaceId || record.kind !== existing.kind ||
|
|
24
|
+
record.ownerKey !== existing.ownerKey || record.principalId !== existing.principalId ||
|
|
25
|
+
record.id !== existing.id || record.binding !== existing.binding || record.createdAt !== existing.createdAt))
|
|
26
|
+
throw new VaultError({ code: "context" });
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=vault.types.js.map
|
package/dist/secrets/testing.js
CHANGED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { type DefaultTreeAdapterMap } from "parse5";
|
|
2
|
+
type Element = DefaultTreeAdapterMap["element"];
|
|
3
|
+
export interface PageLink {
|
|
4
|
+
href: string;
|
|
5
|
+
text: string;
|
|
6
|
+
internal: boolean;
|
|
7
|
+
}
|
|
8
|
+
export interface PageImage {
|
|
9
|
+
src: string;
|
|
10
|
+
alt: string;
|
|
11
|
+
}
|
|
12
|
+
export interface HtmlDocumentFacts {
|
|
13
|
+
title?: string;
|
|
14
|
+
description?: string;
|
|
15
|
+
lang?: string;
|
|
16
|
+
links: PageLink[];
|
|
17
|
+
linksTruncated: boolean;
|
|
18
|
+
images: PageImage[];
|
|
19
|
+
imagesTruncated: boolean;
|
|
20
|
+
stylesheets: string[];
|
|
21
|
+
meta: Record<string, string>;
|
|
22
|
+
}
|
|
23
|
+
export declare const MAX_LINKS = 500;
|
|
24
|
+
export declare const MAX_IMAGES = 300;
|
|
25
|
+
export declare const LINK_PROTOCOLS: ReadonlySet<string>;
|
|
26
|
+
export declare const ASSET_PROTOCOLS: ReadonlySet<string>;
|
|
27
|
+
export declare function attr({ element, name }: {
|
|
28
|
+
element: Element;
|
|
29
|
+
name: string;
|
|
30
|
+
}, _optional?: Record<string, never>): string | undefined;
|
|
31
|
+
/** Resolves against `base`; drops fragment-only, unparseable and other-protocol values, and fragments. */
|
|
32
|
+
export declare function absolute({ raw, base, protocols }: {
|
|
33
|
+
raw: string | undefined;
|
|
34
|
+
base: string;
|
|
35
|
+
protocols: ReadonlySet<string>;
|
|
36
|
+
}, _optional?: Record<string, never>): string | undefined;
|
|
37
|
+
/** First usable image URL: lazy loaders park the real one in `data-src` behind a `data:` placeholder. */
|
|
38
|
+
export declare function imageSource({ img }: {
|
|
39
|
+
img: Element;
|
|
40
|
+
}, _optional?: Record<string, never>): string | undefined;
|
|
41
|
+
/**
|
|
42
|
+
* Extracts page facts. Relative URLs resolve against `<base href>` (itself resolved against the page
|
|
43
|
+
* URL) so links match what a browser would follow.
|
|
44
|
+
* @param required.html - Decoded page source; untrusted.
|
|
45
|
+
* @param required.pageUrl - The final URL the HTML was served from (after redirects).
|
|
46
|
+
* @returns Facts with every URL absolute; links deduped (fragment-free) and capped at {@link MAX_LINKS},
|
|
47
|
+
* images at {@link MAX_IMAGES}; `internal` ignores a leading `www.`.
|
|
48
|
+
* @complexity O(nodes) plus O(anchor subtree) per anchor for its text.
|
|
49
|
+
*/
|
|
50
|
+
export declare function readHtmlDocument({ html, pageUrl }: {
|
|
51
|
+
html: string;
|
|
52
|
+
pageUrl: string;
|
|
53
|
+
}, _optional?: Record<string, never>): HtmlDocumentFacts;
|
|
54
|
+
/**
|
|
55
|
+
* Visible text with one line per block element; scripts, styles, `<head>`, SVG and embeds omitted.
|
|
56
|
+
* @complexity O(nodes).
|
|
57
|
+
*/
|
|
58
|
+
export declare function htmlToText({ html }: {
|
|
59
|
+
html: string;
|
|
60
|
+
}, _optional?: Record<string, never>): string;
|
|
61
|
+
/**
|
|
62
|
+
* The page HTML with `<script>`, `<noscript>`, `<template>`, comments, `on*` handler attributes and
|
|
63
|
+
* `javascript:` attribute values removed (styles and classes kept: theming needs them). Re-serialized by parse5, so it is well-formed rather than byte-identical.
|
|
64
|
+
* @complexity O(nodes).
|
|
65
|
+
*/
|
|
66
|
+
export declare function stripActiveHtml({ html }: {
|
|
67
|
+
html: string;
|
|
68
|
+
}, _optional?: Record<string, never>): string;
|
|
69
|
+
export {};
|
|
70
|
+
//# sourceMappingURL=html-document.d.ts.map
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
import { parse, serialize } from "parse5";
|
|
2
|
+
export const MAX_LINKS = 500;
|
|
3
|
+
export const MAX_IMAGES = 300;
|
|
4
|
+
const MAX_STYLESHEETS = 100;
|
|
5
|
+
const MAX_META = 60;
|
|
6
|
+
const MAX_LINK_TEXT = 200;
|
|
7
|
+
const MAX_META_VALUE = 500;
|
|
8
|
+
export const LINK_PROTOCOLS = new Set(["http:", "https:", "mailto:", "tel:"]);
|
|
9
|
+
export const ASSET_PROTOCOLS = new Set(["http:", "https:"]);
|
|
10
|
+
/** Never visible and never page content; removed from the text and the returned HTML copy. */
|
|
11
|
+
const STRIPPED = new Set(["script", "noscript", "template"]);
|
|
12
|
+
/** Not visible text either, but `style` stays in the HTML copy because theming needs it. */
|
|
13
|
+
const INVISIBLE = new Set([...STRIPPED, "style", "head", "svg", "iframe", "object", "canvas"]);
|
|
14
|
+
const BLOCKS = new Set(["address", "article", "aside", "blockquote", "br", "dd", "div", "dl", "dt", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "ol", "p", "pre", "section", "table", "td", "th", "tr", "ul"]);
|
|
15
|
+
/** `meta name=` keys worth returning; every `og:*` property is returned as well. */
|
|
16
|
+
const META_NAMES = new Set(["generator", "theme-color", "author", "keywords", "twitter:title", "twitter:description", "twitter:image", "application-name"]);
|
|
17
|
+
export function attr({ element, name }, _optional = {}) {
|
|
18
|
+
return element.attrs.find(a => a.name === name)?.value;
|
|
19
|
+
}
|
|
20
|
+
function isElement(node) {
|
|
21
|
+
return "tagName" in node;
|
|
22
|
+
}
|
|
23
|
+
/** Iterative pre-order walk: deeply nested hostile HTML cannot overflow the stack. O(nodes). */
|
|
24
|
+
function* walk(root) {
|
|
25
|
+
const pending = [root];
|
|
26
|
+
while (pending.length) {
|
|
27
|
+
const node = pending.pop();
|
|
28
|
+
yield node;
|
|
29
|
+
// Read after the yield on purpose: stripActiveHtml filters childNodes in between.
|
|
30
|
+
const children = "childNodes" in node ? node.childNodes : [];
|
|
31
|
+
for (let i = children.length - 1; i >= 0; i--)
|
|
32
|
+
pending.push(children[i]);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
function collapse(text) {
|
|
36
|
+
return text.replace(/\s+/g, " ").trim();
|
|
37
|
+
}
|
|
38
|
+
function textOf(root) {
|
|
39
|
+
const parts = [];
|
|
40
|
+
for (const node of walk(root))
|
|
41
|
+
if (node.nodeName === "#text" && "value" in node)
|
|
42
|
+
parts.push(node.value);
|
|
43
|
+
return collapse(parts.join(" "));
|
|
44
|
+
}
|
|
45
|
+
/** Resolves against `base`; drops fragment-only, unparseable and other-protocol values, and fragments. */
|
|
46
|
+
export function absolute({ raw, base, protocols }, _optional = {}) {
|
|
47
|
+
const value = raw?.trim();
|
|
48
|
+
if (!value || value.startsWith("#"))
|
|
49
|
+
return undefined;
|
|
50
|
+
let url;
|
|
51
|
+
try {
|
|
52
|
+
url = new URL(value, base);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return undefined;
|
|
56
|
+
}
|
|
57
|
+
if (!protocols.has(url.protocol))
|
|
58
|
+
return undefined;
|
|
59
|
+
if (url.protocol === "http:" || url.protocol === "https:")
|
|
60
|
+
url.hash = "";
|
|
61
|
+
return url.href;
|
|
62
|
+
}
|
|
63
|
+
function siteHost(hostname) {
|
|
64
|
+
return hostname.toLowerCase().replace(/^www\./, "");
|
|
65
|
+
}
|
|
66
|
+
function isInternal(href, pageHost) {
|
|
67
|
+
const url = new URL(href);
|
|
68
|
+
return (url.protocol === "http:" || url.protocol === "https:") && siteHost(url.hostname) === pageHost;
|
|
69
|
+
}
|
|
70
|
+
function linkText(anchor) {
|
|
71
|
+
const text = textOf(anchor) || attr({ element: anchor, name: "aria-label" }) || attr({ element: anchor, name: "title" }) || "";
|
|
72
|
+
if (text)
|
|
73
|
+
return collapse(text).slice(0, MAX_LINK_TEXT);
|
|
74
|
+
for (const node of walk(anchor))
|
|
75
|
+
if (isElement(node) && node.tagName === "img")
|
|
76
|
+
return collapse(attr({ element: node, name: "alt" }) ?? "").slice(0, MAX_LINK_TEXT);
|
|
77
|
+
return "";
|
|
78
|
+
}
|
|
79
|
+
/** First usable image URL: lazy loaders park the real one in `data-src` behind a `data:` placeholder. */
|
|
80
|
+
export function imageSource({ img }, _optional = {}) {
|
|
81
|
+
const src = attr({ element: img, name: "src" });
|
|
82
|
+
if (src && !src.trim().startsWith("data:"))
|
|
83
|
+
return src;
|
|
84
|
+
return attr({ element: img, name: "data-src" }) ?? attr({ element: img, name: "data-lazy-src" }) ?? attr({ element: img, name: "srcset" })?.trim().split(/\s+/)[0];
|
|
85
|
+
}
|
|
86
|
+
function collect(document) {
|
|
87
|
+
const raw = { anchors: [], images: [], linkTags: [], metas: [] };
|
|
88
|
+
for (const node of walk(document)) {
|
|
89
|
+
if (!isElement(node))
|
|
90
|
+
continue;
|
|
91
|
+
switch (node.tagName) {
|
|
92
|
+
case "a":
|
|
93
|
+
raw.anchors.push(node);
|
|
94
|
+
break;
|
|
95
|
+
case "img":
|
|
96
|
+
raw.images.push(node);
|
|
97
|
+
break;
|
|
98
|
+
case "link":
|
|
99
|
+
raw.linkTags.push(node);
|
|
100
|
+
break;
|
|
101
|
+
case "meta":
|
|
102
|
+
raw.metas.push(node);
|
|
103
|
+
break;
|
|
104
|
+
case "title":
|
|
105
|
+
raw.title ??= node;
|
|
106
|
+
break;
|
|
107
|
+
case "html":
|
|
108
|
+
raw.html ??= node;
|
|
109
|
+
break;
|
|
110
|
+
case "base":
|
|
111
|
+
raw.base ??= attr({ element: node, name: "href" });
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return raw;
|
|
116
|
+
}
|
|
117
|
+
function readMeta(raw, base) {
|
|
118
|
+
const meta = {};
|
|
119
|
+
const put = (key, value) => {
|
|
120
|
+
if (value === undefined || key in meta || Object.keys(meta).length >= MAX_META)
|
|
121
|
+
return;
|
|
122
|
+
meta[key] = collapse(value).slice(0, MAX_META_VALUE);
|
|
123
|
+
};
|
|
124
|
+
for (const element of raw.metas) {
|
|
125
|
+
const property = attr({ element, name: "property" })?.toLowerCase();
|
|
126
|
+
const name = attr({ element, name: "name" })?.toLowerCase();
|
|
127
|
+
const content = attr({ element, name: "content" });
|
|
128
|
+
if (property?.startsWith("og:"))
|
|
129
|
+
put(property, property === "og:image" || property === "og:url" ? absolute({ raw: content, base, protocols: ASSET_PROTOCOLS }) : content);
|
|
130
|
+
else if (name && META_NAMES.has(name))
|
|
131
|
+
put(name, name === "twitter:image" ? absolute({ raw: content, base, protocols: ASSET_PROTOCOLS }) : content);
|
|
132
|
+
}
|
|
133
|
+
for (const element of raw.linkTags) {
|
|
134
|
+
const rel = (attr({ element, name: "rel" }) ?? "").toLowerCase().split(/\s+/);
|
|
135
|
+
const href = absolute({ raw: attr({ element, name: "href" }), base, protocols: ASSET_PROTOCOLS });
|
|
136
|
+
if (rel.includes("canonical"))
|
|
137
|
+
put("canonical", href);
|
|
138
|
+
if (rel.includes("icon"))
|
|
139
|
+
put("favicon", href);
|
|
140
|
+
if (rel.includes("apple-touch-icon"))
|
|
141
|
+
put("apple-touch-icon", href);
|
|
142
|
+
}
|
|
143
|
+
return meta;
|
|
144
|
+
}
|
|
145
|
+
function readLinks(anchors, base, pageHost) {
|
|
146
|
+
const seen = new Set();
|
|
147
|
+
const links = [];
|
|
148
|
+
let truncated = false;
|
|
149
|
+
for (const anchor of anchors) {
|
|
150
|
+
const href = absolute({ raw: attr({ element: anchor, name: "href" }), base, protocols: LINK_PROTOCOLS });
|
|
151
|
+
if (!href || seen.has(href))
|
|
152
|
+
continue;
|
|
153
|
+
seen.add(href);
|
|
154
|
+
if (links.length === MAX_LINKS) {
|
|
155
|
+
truncated = true;
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
links.push({ href, text: linkText(anchor), internal: isInternal(href, pageHost) });
|
|
159
|
+
}
|
|
160
|
+
return { links, truncated };
|
|
161
|
+
}
|
|
162
|
+
function readImages(images, base) {
|
|
163
|
+
const seen = new Set();
|
|
164
|
+
const out = [];
|
|
165
|
+
let truncated = false;
|
|
166
|
+
for (const img of images) {
|
|
167
|
+
const src = absolute({ raw: imageSource({ img }), base, protocols: ASSET_PROTOCOLS });
|
|
168
|
+
if (!src || seen.has(src))
|
|
169
|
+
continue;
|
|
170
|
+
seen.add(src);
|
|
171
|
+
if (out.length === MAX_IMAGES) {
|
|
172
|
+
truncated = true;
|
|
173
|
+
break;
|
|
174
|
+
}
|
|
175
|
+
out.push({ src, alt: collapse(attr({ element: img, name: "alt" }) ?? "") });
|
|
176
|
+
}
|
|
177
|
+
return { images: out, truncated };
|
|
178
|
+
}
|
|
179
|
+
function readStylesheets(linkTags, base) {
|
|
180
|
+
const out = new Set();
|
|
181
|
+
for (const element of linkTags) {
|
|
182
|
+
const rel = (attr({ element, name: "rel" }) ?? "").toLowerCase().split(/\s+/);
|
|
183
|
+
const href = absolute({ raw: attr({ element, name: "href" }), base, protocols: ASSET_PROTOCOLS });
|
|
184
|
+
if (rel.includes("stylesheet") && href && out.size < MAX_STYLESHEETS)
|
|
185
|
+
out.add(href);
|
|
186
|
+
}
|
|
187
|
+
return [...out];
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Extracts page facts. Relative URLs resolve against `<base href>` (itself resolved against the page
|
|
191
|
+
* URL) so links match what a browser would follow.
|
|
192
|
+
* @param required.html - Decoded page source; untrusted.
|
|
193
|
+
* @param required.pageUrl - The final URL the HTML was served from (after redirects).
|
|
194
|
+
* @returns Facts with every URL absolute; links deduped (fragment-free) and capped at {@link MAX_LINKS},
|
|
195
|
+
* images at {@link MAX_IMAGES}; `internal` ignores a leading `www.`.
|
|
196
|
+
* @complexity O(nodes) plus O(anchor subtree) per anchor for its text.
|
|
197
|
+
*/
|
|
198
|
+
export function readHtmlDocument({ html, pageUrl }, _optional = {}) {
|
|
199
|
+
const document = parse(html);
|
|
200
|
+
const raw = collect(document);
|
|
201
|
+
const base = absolute({ raw: raw.base, base: pageUrl, protocols: ASSET_PROTOCOLS }) ?? pageUrl;
|
|
202
|
+
const pageHost = siteHost(new URL(pageUrl).hostname);
|
|
203
|
+
const links = readLinks(raw.anchors, base, pageHost);
|
|
204
|
+
const images = readImages(raw.images, base);
|
|
205
|
+
const description = raw.metas.find(m => attr({ element: m, name: "name" })?.toLowerCase() === "description");
|
|
206
|
+
const title = raw.title ? textOf(raw.title) : "";
|
|
207
|
+
const lang = raw.html ? attr({ element: raw.html, name: "lang" })?.trim() : undefined;
|
|
208
|
+
const descriptionText = description ? collapse(attr({ element: description, name: "content" }) ?? "") : "";
|
|
209
|
+
return {
|
|
210
|
+
...(title ? { title } : {}),
|
|
211
|
+
...(descriptionText ? { description: descriptionText } : {}),
|
|
212
|
+
...(lang ? { lang } : {}),
|
|
213
|
+
links: links.links, linksTruncated: links.truncated,
|
|
214
|
+
images: images.images, imagesTruncated: images.truncated,
|
|
215
|
+
stylesheets: readStylesheets(raw.linkTags, base),
|
|
216
|
+
meta: readMeta(raw, base),
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Visible text with one line per block element; scripts, styles, `<head>`, SVG and embeds omitted.
|
|
221
|
+
* @complexity O(nodes).
|
|
222
|
+
*/
|
|
223
|
+
export function htmlToText({ html }, _optional = {}) {
|
|
224
|
+
const pending = [parse(html)];
|
|
225
|
+
const out = [];
|
|
226
|
+
while (pending.length) {
|
|
227
|
+
const node = pending.pop();
|
|
228
|
+
if (node === "\n") {
|
|
229
|
+
out.push("\n");
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
if (isElement(node) && INVISIBLE.has(node.tagName))
|
|
233
|
+
continue;
|
|
234
|
+
if (node.nodeName === "#text" && "value" in node) {
|
|
235
|
+
out.push(node.value);
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
if (!("childNodes" in node))
|
|
239
|
+
continue;
|
|
240
|
+
const block = isElement(node) && BLOCKS.has(node.tagName);
|
|
241
|
+
if (block) {
|
|
242
|
+
out.push("\n");
|
|
243
|
+
pending.push("\n");
|
|
244
|
+
}
|
|
245
|
+
for (let i = node.childNodes.length - 1; i >= 0; i--)
|
|
246
|
+
pending.push(node.childNodes[i]);
|
|
247
|
+
}
|
|
248
|
+
return out.join("").split("\n").map(line => line.replace(/[^\S\n]+/g, " ").trim()).filter(Boolean).join("\n");
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* The page HTML with `<script>`, `<noscript>`, `<template>`, comments, `on*` handler attributes and
|
|
252
|
+
* `javascript:` attribute values removed (styles and classes kept: theming needs them). Re-serialized by parse5, so it is well-formed rather than byte-identical.
|
|
253
|
+
* @complexity O(nodes).
|
|
254
|
+
*/
|
|
255
|
+
export function stripActiveHtml({ html }, _optional = {}) {
|
|
256
|
+
const document = parse(html);
|
|
257
|
+
for (const node of walk(document)) {
|
|
258
|
+
if (!("childNodes" in node))
|
|
259
|
+
continue;
|
|
260
|
+
const parent = node;
|
|
261
|
+
parent.childNodes = parent.childNodes.filter(child => child.nodeName !== "#comment" && !(isElement(child) && STRIPPED.has(child.tagName)));
|
|
262
|
+
// Inert copy: an agent may paste this into a page, so handlers and script URLs never survive.
|
|
263
|
+
if (isElement(node))
|
|
264
|
+
node.attrs = node.attrs.filter(a => !/^on/i.test(a.name) && !/^\s*javascript:/i.test(a.value));
|
|
265
|
+
}
|
|
266
|
+
return serialize(document);
|
|
267
|
+
}
|
|
268
|
+
//# sourceMappingURL=html-document.js.map
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Bound synchronous HTML parsers independently of the HTTP response byte limit. */
|
|
2
|
+
export declare const MAX_HTML_CHARS = 1000000;
|
|
3
|
+
/** Return a bounded prefix without splitting UTF-16; fetch projections expose its truncation flag.
|
|
4
|
+
* The caller may supply a lower projection budget; direct conversion always has a fixed ceiling.
|
|
5
|
+
* O(1) decision plus O(cap) prefix allocation; raw body output keeps its separate transport cap.
|
|
6
|
+
*/
|
|
7
|
+
export declare function boundHtml({ html }: {
|
|
8
|
+
html: string;
|
|
9
|
+
}, { maxChars }?: {
|
|
10
|
+
maxChars?: number;
|
|
11
|
+
}): {
|
|
12
|
+
html: string;
|
|
13
|
+
truncated: boolean;
|
|
14
|
+
};
|
|
15
|
+
//# sourceMappingURL=html-resource.d.ts.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Bound synchronous HTML parsers independently of the HTTP response byte limit. */
|
|
2
|
+
export const MAX_HTML_CHARS = 1_000_000;
|
|
3
|
+
/** Return a bounded prefix without splitting UTF-16; fetch projections expose its truncation flag.
|
|
4
|
+
* The caller may supply a lower projection budget; direct conversion always has a fixed ceiling.
|
|
5
|
+
* O(1) decision plus O(cap) prefix allocation; raw body output keeps its separate transport cap.
|
|
6
|
+
*/
|
|
7
|
+
export function boundHtml({ html }, { maxChars = MAX_HTML_CHARS } = {}) {
|
|
8
|
+
const limit = Math.min(MAX_HTML_CHARS, maxChars);
|
|
9
|
+
const end = /[\uDC00-\uDFFF]/.test(html[limit] ?? "") ? limit - 1 : limit;
|
|
10
|
+
return html.length > limit ? { html: html.slice(0, end), truncated: true } : { html, truncated: false };
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=html-resource.js.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert untrusted HTML to readable Markdown with absolute, allowlisted link/image URLs.
|
|
3
|
+
* @returns Markdown without surrounding whitespace; input is capped before parsing and errors propagate.
|
|
4
|
+
* @complexity O(nodes + output size) time/space for ordinary pages; nesting is bounded at 300.
|
|
5
|
+
* @example htmlToMarkdown({ html: "<h1>Hello</h1>", pageUrl: "https://example.com/" })
|
|
6
|
+
*/
|
|
7
|
+
export declare function htmlToMarkdown({ html, pageUrl }: {
|
|
8
|
+
html: string;
|
|
9
|
+
pageUrl: string;
|
|
10
|
+
}, _optional?: Record<string, never>): string;
|
|
11
|
+
//# sourceMappingURL=html-to-markdown.d.ts.map
|