@jini-ai/platform 0.5.8 → 0.5.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/asset-cache.js +26 -25
  3. package/dist/blob-storage.d.ts +7 -2
  4. package/dist/blob-storage.js +99 -59
  5. package/dist/child-environment.d.ts +36 -0
  6. package/dist/child-environment.js +68 -0
  7. package/dist/download.js +192 -148
  8. package/dist/fs/atomic-write.d.ts +25 -2
  9. package/dist/fs/atomic-write.js +162 -111
  10. package/dist/fs/boot-identity.d.ts +16 -0
  11. package/dist/fs/boot-identity.js +38 -17
  12. package/dist/fs/directory-home.js +6 -0
  13. package/dist/fs/directory-registry.d.ts +1 -1
  14. package/dist/fs/directory-registry.js +23 -16
  15. package/dist/fs/directory-running-lock.d.ts +1 -1
  16. package/dist/fs/directory-running-lock.js +18 -7
  17. package/dist/fs/directory-sync.d.ts +1 -1
  18. package/dist/fs/directory-sync.js +64 -51
  19. package/dist/fs/directory-user-config.js +108 -67
  20. package/dist/fs/directory-workspace.d.ts +11 -0
  21. package/dist/fs/directory-workspace.js +16 -11
  22. package/dist/fs/durable-json.d.ts +3 -1
  23. package/dist/fs/durable-json.js +55 -40
  24. package/dist/fs/file-lock.js +214 -149
  25. package/dist/fs/guarded-reader.js +47 -35
  26. package/dist/fs/user-data-directories.d.ts +1 -1
  27. package/dist/fs/user-data-directories.js +25 -9
  28. package/dist/fs/verified-directory-move.d.ts +14 -11
  29. package/dist/fs/verified-directory-move.js +106 -80
  30. package/dist/fs/verified-file-relocation.d.ts +2 -1
  31. package/dist/fs/verified-file-relocation.js +112 -100
  32. package/dist/http/guarded/client.d.ts +3 -4
  33. package/dist/http/guarded/client.js +50 -60
  34. package/dist/http/guarded/fetch-adapter.d.ts +2 -3
  35. package/dist/http/guarded/fetch-adapter.js +26 -15
  36. package/dist/mail/builtin-providers.d.ts +39 -0
  37. package/dist/mail/builtin-providers.js +60 -0
  38. package/dist/mail/feedback.d.ts +84 -0
  39. package/dist/mail/feedback.js +90 -0
  40. package/dist/mail/index.d.ts +11 -0
  41. package/dist/mail/index.js +6 -0
  42. package/dist/mail/policy.d.ts +77 -0
  43. package/dist/mail/policy.js +155 -0
  44. package/dist/mail/ports.d.ts +56 -7
  45. package/dist/mail/provider.d.ts +117 -0
  46. package/dist/mail/provider.js +74 -0
  47. package/dist/mail/rate-limit.d.ts +33 -0
  48. package/dist/mail/rate-limit.js +35 -0
  49. package/dist/mail/sender-identity.d.ts +68 -0
  50. package/dist/mail/sender-identity.js +50 -0
  51. package/dist/mail/types.d.ts +9 -0
  52. package/dist/net/address.d.ts +9 -23
  53. package/dist/net/address.js +28 -133
  54. package/dist/net/endpoint-policy.d.ts +26 -2
  55. package/dist/net/endpoint-policy.js +159 -56
  56. package/dist/object-store/index.d.ts +2 -0
  57. package/dist/object-store/index.js +2 -0
  58. package/dist/object-store/local-fs.d.ts +17 -0
  59. package/dist/object-store/local-fs.js +146 -0
  60. package/dist/process.js +2 -0
  61. package/dist/proxy-env.js +131 -135
  62. package/dist/secrets/credential-sets/contracts.d.ts +1 -0
  63. package/dist/secrets/credential-sets/contracts.js +1 -0
  64. package/dist/secrets/credential-sets/index.d.ts +1 -0
  65. package/dist/secrets/credential-sets/index.js +1 -0
  66. package/dist/secrets/credential-sets/provider-matches.d.ts +61 -0
  67. package/dist/secrets/credential-sets/provider-matches.js +61 -0
  68. package/dist/secrets/credential-sets/types.d.ts +7 -0
  69. package/dist/secrets/credential-sets/vault.meta.js +58 -20
  70. package/dist/secrets/credential-sets/vault.repo.memory.d.ts +7 -0
  71. package/dist/secrets/credential-sets/vault.repo.memory.js +43 -36
  72. package/dist/secrets/credential-sets/vault.service.d.ts +9 -0
  73. package/dist/secrets/credential-sets/vault.service.js +131 -61
  74. package/dist/secrets/credential-sets/vault.types.d.ts +11 -3
  75. package/dist/secrets/credential-sets/vault.types.js +22 -8
  76. package/dist/secrets/keyring.env.js +14 -10
  77. package/dist/toolchain.js +119 -91
  78. package/dist/web/html-document.js +73 -69
  79. package/dist/web/html-to-markdown.js +127 -102
  80. package/dist/web/web-page.d.ts +1 -1
  81. package/dist/web/web-page.js +162 -81
  82. package/package.json +11 -3
@@ -24,15 +24,7 @@ export class EnvOrFileKeyring {
24
24
  hkdfSalt;
25
25
  cachedRootKey;
26
26
  constructor(required, optional = {}) {
27
- requireHostOption(required?.envVarName, "envVarName");
28
- requireHostOption(required?.keyFilePath, "keyFilePath");
29
- requireHostOption(required?.hkdfSalt, "hkdfSalt");
30
- if (typeof required.env?.read !== "function")
31
- throw new TypeError("env.read is required");
32
- if (typeof required.allowFileFallback !== "boolean")
33
- throw new TypeError("allowFileFallback must be an explicit boolean");
34
- if (typeof required.allowFileAutoGenerate !== "boolean")
35
- throw new TypeError("allowFileAutoGenerate must be an explicit boolean");
27
+ assertEnvOrFileKeyringRequired({ required });
36
28
  this.messages = optional.messages ?? defaultPlatformMessages;
37
29
  this.env = required.env;
38
30
  this.hkdfSalt = required.hkdfSalt;
@@ -178,7 +170,7 @@ function unusableRootKeyMessage(input) {
178
170
  const refusal = `${input.subject} is not usable as a root key: ${input.detail}. The keyring refuses to derive any key material from it rather than sealing under a shortened or empty key.`;
179
171
  if (input.sealedWarningSubject === undefined)
180
172
  return refusal;
181
- return (`${refusal} ${(input.messages ?? defaultPlatformMessages).rootKeySealedWarning({ subject: input.sealedWarningSubject })}` +
173
+ return (`${refusal} ${input.messages.rootKeySealedWarning({ subject: input.sealedWarningSubject })}` +
182
174
  `not from a real ${ROOT_KEY_LENGTH_BYTES}-byte key, so those stored credentials are not protected as intended — with empty or very short material the derived key is computable from published constants. ` +
183
175
  `Replacing or regenerating the key will NOT restore them; it will make them unreadable. ` +
184
176
  `Do not overwrite, delete or regenerate this key until you have decided what to do with the credentials already stored.`);
@@ -264,6 +256,18 @@ function isAlreadyExistsError(err) {
264
256
  const code = err.code;
265
257
  return code === "EEXIST" || code === "ELOOP";
266
258
  }
259
+ /** Every source permission is explicit; checked in order so the first missing option is named. */
260
+ function assertEnvOrFileKeyringRequired({ required }) {
261
+ requireHostOption(required?.envVarName, "envVarName");
262
+ requireHostOption(required?.keyFilePath, "keyFilePath");
263
+ requireHostOption(required?.hkdfSalt, "hkdfSalt");
264
+ if (typeof required.env?.read !== "function")
265
+ throw new TypeError("env.read is required");
266
+ if (typeof required.allowFileFallback !== "boolean")
267
+ throw new TypeError("allowFileFallback must be an explicit boolean");
268
+ if (typeof required.allowFileAutoGenerate !== "boolean")
269
+ throw new TypeError("allowFileAutoGenerate must be an explicit boolean");
270
+ }
267
271
  function requireHostOption(value, name) {
268
272
  if (value === undefined || value === null)
269
273
  throw new TypeError(`${name} is required`);
package/dist/toolchain.js CHANGED
@@ -25,6 +25,14 @@ function resolveUserScopedHome(raw, home) {
25
25
  }
26
26
  return isAbsolute(value) ? value : null;
27
27
  }
28
+ /** @internal Trim an optional env value, treating a missing/non-string value as blank. */
29
+ function trimmedEnv(value) {
30
+ return typeof value === "string" ? value.trim() : "";
31
+ }
32
+ /** @internal Whether the current process runs on Windows (read live so tests can swap `process.platform`). */
33
+ function isWindows() {
34
+ return process.platform === "win32";
35
+ }
28
36
  // Single source of truth for "user-level CLI install locations the daemon
29
37
  // must search even when launched with a minimal PATH". GUI launchers
30
38
  // (macOS .app bundles, Linux .desktop files) typically inherit a stripped
@@ -46,7 +54,7 @@ function resolveUserScopedHome(raw, home) {
46
54
  */
47
55
  export function wellKnownUserToolchainBins(options = {}) {
48
56
  const home = options.home ?? homedir();
49
- const includeSystemBins = options.includeSystemBins ?? process.platform !== "win32";
57
+ const includeSystemBins = options.includeSystemBins ?? !isWindows();
50
58
  const env = options.env ?? process.env;
51
59
  const dirs = [];
52
60
  // Vite+ global installs expose CLI shims from VP_HOME/bin (default
@@ -57,6 +65,40 @@ export function wellKnownUserToolchainBins(options = {}) {
57
65
  if (vpHome) {
58
66
  dirs.push(join(vpHome, "bin"));
59
67
  }
68
+ dirs.push(...npmPrefixBins({ env, home }));
69
+ dirs.push(...conventionalUserBins({ home }));
70
+ dirs.push(...windowsUserBins({ env, home }));
71
+ // Mise shims: makes every tool installed with `mise install` visible to
72
+ // GUI-launched daemons even when the process inherits a stripped PATH.
73
+ // Respect MISE_DATA_DIR (the official way to relocate the whole mise tree).
74
+ // We only fall back to the legacy ~/.mise/shims path when no explicit
75
+ // MISE_DATA_DIR override is provided.
76
+ const miseDataOverride = resolveUserScopedHome(env.MISE_DATA_DIR, home);
77
+ const miseData = miseDataOverride ?? join(home, ".local", "share", "mise");
78
+ dirs.push(join(miseData, "shims"));
79
+ if (!miseDataOverride) {
80
+ dirs.push(join(home, ".mise", "shims"));
81
+ }
82
+ if (includeSystemBins) {
83
+ dirs.push("/opt/homebrew/bin", "/usr/local/bin");
84
+ }
85
+ // Per-version Node toolchains: scan the install root and surface every
86
+ // version directory's bin folder. Best-effort — missing roots simply
87
+ // contribute nothing.
88
+ // When MISE_DATA_DIR is set we use the same root for consistency with shims.
89
+ const miseInstalls = join(miseData, "installs");
90
+ // Surface bin dirs for npm-packaged toolchains installed under the mise
91
+ // installs root (e.g. `npm-openai-codex`).
92
+ dirs.push(...existingChildBinDirs(join(miseInstalls, "npm-openai-codex"), ["bin"]));
93
+ for (const installRoot of nodeInstallRoots({ env, home, miseInstalls })) {
94
+ dirs.push(...existingChildBinDirs(installRoot.root, installRoot.segments));
95
+ }
96
+ return dirs;
97
+ }
98
+ /** @internal The npm-prefix bins derived from the env plus the Windows `%APPDATA%\npm` default. */
99
+ function npmPrefixBins(input) {
100
+ const { env, home } = input;
101
+ const dirs = [];
60
102
  // The user's *explicit* npm prefix outranks every conventional
61
103
  // location below — including `~/.local/bin`. The env var is the
62
104
  // user's current npm configuration, so a binary installed via
@@ -72,21 +114,18 @@ export function wellKnownUserToolchainBins(options = {}) {
72
114
  // Trim before length-checking so accidental whitespace-only values
73
115
  // (`NPM_CONFIG_PREFIX=" "`) do not produce a `/bin`-suffixed garbage
74
116
  // entry.
75
- const npmPrefixRaw = env.NPM_CONFIG_PREFIX ?? env.npm_config_prefix;
76
- if (typeof npmPrefixRaw === "string") {
77
- const npmPrefix = npmPrefixRaw.trim();
78
- if (npmPrefix.length > 0) {
79
- // Unix: npm global binaries live in <prefix>/bin. Always add it as a
80
- // search-path candidate — existence is irrelevant for a PATH directory,
81
- // and this preserves the long-standing helper contract.
82
- dirs.push(join(npmPrefix, "bin"));
83
- // Windows: npm installs global binaries directly into the prefix root
84
- // (there is no /bin subdirectory), so add the root as well. Additive —
85
- // the <prefix>/bin entry above is left untouched for cross-platform
86
- // parity.
87
- if (process.platform === "win32") {
88
- dirs.push(npmPrefix);
89
- }
117
+ const npmPrefix = trimmedEnv(env.NPM_CONFIG_PREFIX ?? env.npm_config_prefix);
118
+ if (npmPrefix.length > 0) {
119
+ // Unix: npm global binaries live in <prefix>/bin. Always add it as a
120
+ // search-path candidate — existence is irrelevant for a PATH directory,
121
+ // and this preserves the long-standing helper contract.
122
+ dirs.push(join(npmPrefix, "bin"));
123
+ // Windows: npm installs global binaries directly into the prefix root
124
+ // (there is no /bin subdirectory), so add the root as well. Additive —
125
+ // the <prefix>/bin entry above is left untouched for cross-platform
126
+ // parity.
127
+ if (isWindows()) {
128
+ dirs.push(npmPrefix);
90
129
  }
91
130
  }
92
131
  // Windows: %APPDATA%\npm is npm's default global prefix. NPM_CONFIG_PREFIX /
@@ -94,53 +133,59 @@ export function wellKnownUserToolchainBins(options = {}) {
94
133
  // child-process environments, so the block above no-ops for most Windows
95
134
  // users. Add the default location unconditionally — consistent with the
96
135
  // conventional dirs below, which are also added without an existence check.
97
- if (process.platform === "win32") {
136
+ if (isWindows()) {
98
137
  dirs.push(join(home, "AppData", "Roaming", "npm"));
99
138
  }
100
- dirs.push(join(home, ".local", "bin"), join(home, ".vite-plus", "bin"), join(home, ".kimi-code", "bin"), join(home, ".opencode", "bin"), join(home, ".bun", "bin"), join(home, ".volta", "bin"), join(home, ".asdf", "shims"), join(home, "Library", "pnpm"), join(home, ".cargo", "bin"),
101
- // Common user-level npm prefixes for sudo-free global installs.
102
- // ~/.npm-global is the dominant non-canonical convention shipped
103
- // in most third-party "fix npm EACCES" tutorials, and
104
- // ~/.npm-packages is the second-most common variant. Without
105
- // these, GUI-launched daemons miss `npm i -g`'d CLIs even though
106
- // they resolve cleanly from the user's shell.
107
- join(home, ".npm-global", "bin"), join(home, ".npm-packages", "bin"),
108
- // Other common user-level toolchains that install CLI shims outside the
109
- // Node ecosystem but still ship agent CLIs (or their dependencies):
110
- // Deno's install root, Go's default GOBIN, and pyenv's shim dir. All are
111
- // best-effort — a missing dir contributes nothing.
112
- join(home, ".deno", "bin"), join(home, "go", "bin"), join(home, ".pyenv", "shims"));
139
+ return dirs;
140
+ }
141
+ /** @internal Conventional home-relative toolchain bin dirs, added without an existence check. */
142
+ function conventionalUserBins(input) {
143
+ const { home } = input;
144
+ return [
145
+ join(home, ".local", "bin"),
146
+ join(home, ".vite-plus", "bin"),
147
+ join(home, ".kimi-code", "bin"),
148
+ join(home, ".opencode", "bin"),
149
+ join(home, ".bun", "bin"),
150
+ join(home, ".volta", "bin"),
151
+ join(home, ".asdf", "shims"),
152
+ join(home, "Library", "pnpm"),
153
+ join(home, ".cargo", "bin"),
154
+ // Common user-level npm prefixes for sudo-free global installs.
155
+ // ~/.npm-global is the dominant non-canonical convention shipped
156
+ // in most third-party "fix npm EACCES" tutorials, and
157
+ // ~/.npm-packages is the second-most common variant. Without
158
+ // these, GUI-launched daemons miss `npm i -g`'d CLIs even though
159
+ // they resolve cleanly from the user's shell.
160
+ join(home, ".npm-global", "bin"),
161
+ join(home, ".npm-packages", "bin"),
162
+ // Other common user-level toolchains that install CLI shims outside the
163
+ // Node ecosystem but still ship agent CLIs (or their dependencies):
164
+ // Deno's install root, Go's default GOBIN, and pyenv's shim dir. All are
165
+ // best-effort — a missing dir contributes nothing.
166
+ join(home, ".deno", "bin"),
167
+ join(home, "go", "bin"),
168
+ join(home, ".pyenv", "shims"),
169
+ ];
170
+ }
171
+ /** @internal Windows-only scoop shims and `%APPDATA%\npm`; empty on POSIX. */
172
+ function windowsUserBins(input) {
113
173
  // Windows-only user install roots that GUI launches miss. Scoop drops
114
174
  // shims under ~/scoop/shims, and npm's global prefix on Windows defaults
115
175
  // to %APPDATA%\npm rather than a `bin` subdir.
116
- if (process.platform === "win32") {
117
- dirs.push(join(home, "scoop", "shims"));
118
- const appData = typeof env.APPDATA === "string" ? env.APPDATA.trim() : "";
119
- if (appData.length > 0) {
120
- dirs.push(join(appData, "npm"));
121
- }
176
+ if (!isWindows())
177
+ return [];
178
+ const dirs = [join(input.home, "scoop", "shims")];
179
+ const appData = trimmedEnv(input.env.APPDATA);
180
+ if (appData.length > 0) {
181
+ dirs.push(join(appData, "npm"));
122
182
  }
123
- // Mise shims: makes every tool installed with `mise install` visible to
124
- // GUI-launched daemons even when the process inherits a stripped PATH.
125
- // Respect MISE_DATA_DIR (the official way to relocate the whole mise tree).
126
- // We only fall back to the legacy ~/.mise/shims path when no explicit
127
- // MISE_DATA_DIR override is provided.
128
- const miseDataOverride = resolveUserScopedHome(env.MISE_DATA_DIR, home);
129
- const miseData = miseDataOverride || join(home, ".local", "share", "mise");
130
- dirs.push(join(miseData, "shims"));
131
- if (!miseDataOverride) {
132
- dirs.push(join(home, ".mise", "shims"));
133
- }
134
- if (includeSystemBins) {
135
- dirs.push("/opt/homebrew/bin", "/usr/local/bin");
136
- }
137
- // Per-version Node toolchains: scan the install root and surface every
138
- // version directory's bin folder. Best-effort — missing roots simply
139
- // contribute nothing.
140
- // When MISE_DATA_DIR is set we use the same root for consistency with shims.
141
- const miseInstalls = join(miseData, "installs");
142
- dirs.push(...existingMiseNpmPackageBinDirs(miseInstalls));
143
- const nodeInstallRoots = [
183
+ return dirs;
184
+ }
185
+ /** @internal Per-version Node install roots to probe, in priority order (mise, nvm, fnm, then Windows fnm). */
186
+ function nodeInstallRoots(input) {
187
+ const { env, home, miseInstalls } = input;
188
+ const roots = [
144
189
  {
145
190
  root: join(miseInstalls, "node"),
146
191
  segments: ["bin"],
@@ -166,38 +211,23 @@ export function wellKnownUserToolchainBins(options = {}) {
166
211
  // inherits a stripped PATH and reads no shell rc, so without an explicit
167
212
  // probe fnm-managed Node — and every agent CLI it runs — is silently
168
213
  // undetected on Windows. See issues #3517 and #3062.
169
- if (process.platform === "win32") {
170
- const fnmDirOverride = typeof env.FNM_DIR === "string" ? env.FNM_DIR.trim() : "";
171
- const fnmRoots = [];
172
- if (fnmDirOverride.length > 0) {
173
- fnmRoots.push(fnmDirOverride);
174
- }
175
- else {
176
- for (const base of [env.LOCALAPPDATA, env.APPDATA]) {
177
- const trimmed = typeof base === "string" ? base.trim() : "";
178
- if (trimmed.length > 0)
179
- fnmRoots.push(join(trimmed, "fnm"));
180
- }
181
- }
182
- for (const fnmRoot of fnmRoots) {
183
- nodeInstallRoots.push({ root: join(fnmRoot, "node-versions"), segments: ["installation"] });
214
+ if (isWindows()) {
215
+ for (const fnmRoot of windowsFnmRoots({ env })) {
216
+ roots.push({ root: join(fnmRoot, "node-versions"), segments: ["installation"] });
184
217
  }
185
218
  }
186
- for (const installRoot of nodeInstallRoots) {
187
- for (const dir of existingChildBinDirs(installRoot.root, installRoot.segments)) {
188
- dirs.push(dir);
189
- }
190
- }
191
- return dirs;
219
+ return roots;
192
220
  }
193
- /** @internal Surface bin dirs for npm-packaged toolchains installed under a mise installs root (e.g. `npm-openai-codex`). */
194
- function existingMiseNpmPackageBinDirs(root) {
195
- const out = [];
196
- for (const packageName of ["npm-openai-codex"]) {
197
- const packageRoot = join(root, packageName);
198
- out.push(...existingChildBinDirs(packageRoot, ["bin"]));
199
- }
200
- return out;
221
+ /** @internal Windows fnm roots: `FNM_DIR` when set, else `%LOCALAPPDATA%\fnm` then `%APPDATA%\fnm`. */
222
+ function windowsFnmRoots(input) {
223
+ const { env } = input;
224
+ const fnmDirOverride = trimmedEnv(env.FNM_DIR);
225
+ if (fnmDirOverride.length > 0)
226
+ return [fnmDirOverride];
227
+ return [env.LOCALAPPDATA, env.APPDATA]
228
+ .map(trimmedEnv)
229
+ .filter((base) => base.length > 0)
230
+ .map((base) => join(base, "fnm"));
201
231
  }
202
232
  /** @internal List existing `<root>/<versionDir>/<...segments>` directories, version-sorted, skipping non-directories. */
203
233
  function existingChildBinDirs(root, segments) {
@@ -209,7 +239,9 @@ function existingChildBinDirs(root, segments) {
209
239
  catch {
210
240
  return out;
211
241
  }
212
- for (const entry of sortVersionedDirEntries(entries)) {
242
+ // Sort directory entries by semver-like name (newest first), falling back to locale order.
243
+ const sorted = [...entries].sort((left, right) => compareVersionLikeDirNames(left.name, right.name));
244
+ for (const entry of sorted) {
213
245
  if (!entry.isDirectory() && !entry.isSymbolicLink())
214
246
  continue;
215
247
  const candidate = join(root, entry.name, ...segments);
@@ -218,10 +250,6 @@ function existingChildBinDirs(root, segments) {
218
250
  }
219
251
  return out;
220
252
  }
221
- /** @internal Sort directory entries by semver-like name (newest first), falling back to locale order. */
222
- function sortVersionedDirEntries(entries) {
223
- return [...entries].sort((left, right) => compareVersionLikeDirNames(left.name, right.name));
224
- }
225
253
  /** @internal Compare two version-like directory names descending; semver-parseable names precede non-parseable ones. */
226
254
  function compareVersionLikeDirNames(left, right) {
227
255
  const leftSemver = parseVersionLikeDirName(left);
@@ -83,65 +83,63 @@ export function imageSource({ img }, _optional = {}) {
83
83
  return src;
84
84
  return attr({ element: img, name: "data-src" }) ?? attr({ element: img, name: "data-lazy-src" }) ?? attr({ element: img, name: "srcset" })?.trim().split(/\s+/)[0];
85
85
  }
86
+ /** A Map, not an object literal, so tag names such as `constructor` never hit Object.prototype. */
87
+ const COLLECTED = new Map([["a", "anchors"], ["img", "images"], ["link", "linkTags"], ["meta", "metas"]]);
86
88
  function collect(document) {
87
89
  const raw = { anchors: [], images: [], linkTags: [], metas: [] };
88
90
  for (const node of walk(document)) {
89
91
  if (!isElement(node))
90
92
  continue;
91
- switch (node.tagName) {
92
- case "a":
93
- raw.anchors.push(node);
94
- break;
95
- case "img":
96
- raw.images.push(node);
97
- break;
98
- case "link":
99
- raw.linkTags.push(node);
100
- break;
101
- case "meta":
102
- raw.metas.push(node);
103
- break;
104
- case "title":
105
- raw.title ??= node;
106
- break;
107
- case "html":
108
- raw.html ??= node;
109
- break;
110
- case "base":
111
- raw.base ??= attr({ element: node, name: "href" });
112
- break;
113
- }
93
+ const list = COLLECTED.get(node.tagName);
94
+ if (list)
95
+ raw[list].push(node);
96
+ else if (node.tagName === "title" || node.tagName === "html")
97
+ raw[node.tagName] ??= node;
98
+ // The first base WITH an href wins, as in browsers.
99
+ else if (node.tagName === "base")
100
+ raw.base ??= attr({ element: node, name: "href" });
114
101
  }
115
102
  return raw;
116
103
  }
104
+ /** Meta keys whose values are URLs and are returned absolute. */
105
+ const URL_META = new Set(["og:image", "og:url", "twitter:image"]);
106
+ /** `<link rel>` tokens copied into meta, in emission order, under their meta key. */
107
+ const LINK_META = [["canonical", "canonical"], ["icon", "favicon"], ["apple-touch-icon", "apple-touch-icon"]];
117
108
  function readMeta(raw, base) {
118
109
  const meta = {};
119
- const put = (key, value) => {
120
- if (value === undefined || key in meta || Object.keys(meta).length >= MAX_META)
121
- return;
122
- meta[key] = collapse(value).slice(0, MAX_META_VALUE);
123
- };
124
110
  for (const element of raw.metas) {
125
- const property = attr({ element, name: "property" })?.toLowerCase();
126
- const name = attr({ element, name: "name" })?.toLowerCase();
127
- const content = attr({ element, name: "content" });
128
- if (property?.startsWith("og:"))
129
- put(property, property === "og:image" || property === "og:url" ? absolute({ raw: content, base, protocols: ASSET_PROTOCOLS }) : content);
130
- else if (name && META_NAMES.has(name))
131
- put(name, name === "twitter:image" ? absolute({ raw: content, base, protocols: ASSET_PROTOCOLS }) : content);
111
+ const entry = metaTagEntry({ element, base });
112
+ if (entry)
113
+ putMeta({ meta, key: entry[0], value: entry[1] });
132
114
  }
133
115
  for (const element of raw.linkTags) {
134
- const rel = (attr({ element, name: "rel" }) ?? "").toLowerCase().split(/\s+/);
116
+ const rels = relTokens({ element });
135
117
  const href = absolute({ raw: attr({ element, name: "href" }), base, protocols: ASSET_PROTOCOLS });
136
- if (rel.includes("canonical"))
137
- put("canonical", href);
138
- if (rel.includes("icon"))
139
- put("favicon", href);
140
- if (rel.includes("apple-touch-icon"))
141
- put("apple-touch-icon", href);
118
+ for (const [rel, key] of LINK_META)
119
+ if (rels.includes(rel))
120
+ putMeta({ meta, key, value: href });
142
121
  }
143
122
  return meta;
144
123
  }
124
+ /** First value per key wins; the entry count and each value are capped. */
125
+ function putMeta({ meta, key, value }, _optional = {}) {
126
+ if (value === undefined || key in meta || Object.keys(meta).length >= MAX_META)
127
+ return;
128
+ meta[key] = collapse(value).slice(0, MAX_META_VALUE);
129
+ }
130
+ /** Every `og:*` property, or an allowlisted `name`; URL-valued keys are resolved. */
131
+ function metaTagEntry({ element, base }, _optional = {}) {
132
+ const property = attr({ element, name: "property" })?.toLowerCase();
133
+ const name = attr({ element, name: "name" })?.toLowerCase();
134
+ const key = property?.startsWith("og:") ? property : name && META_NAMES.has(name) ? name : undefined;
135
+ if (key === undefined)
136
+ return undefined;
137
+ const content = attr({ element, name: "content" });
138
+ return [key, URL_META.has(key) ? absolute({ raw: content, base, protocols: ASSET_PROTOCOLS }) : content];
139
+ }
140
+ function relTokens({ element }, _optional = {}) {
141
+ return (attr({ element, name: "rel" }) ?? "").toLowerCase().split(/\s+/);
142
+ }
145
143
  function readLinks(anchors, base, pageHost) {
146
144
  const seen = new Set();
147
145
  const links = [];
@@ -179,9 +177,8 @@ function readImages(images, base) {
179
177
  function readStylesheets(linkTags, base) {
180
178
  const out = new Set();
181
179
  for (const element of linkTags) {
182
- const rel = (attr({ element, name: "rel" }) ?? "").toLowerCase().split(/\s+/);
183
180
  const href = absolute({ raw: attr({ element, name: "href" }), base, protocols: ASSET_PROTOCOLS });
184
- if (rel.includes("stylesheet") && href && out.size < MAX_STYLESHEETS)
181
+ if (relTokens({ element }).includes("stylesheet") && href && out.size < MAX_STYLESHEETS)
185
182
  out.add(href);
186
183
  }
187
184
  return [...out];
@@ -202,20 +199,25 @@ export function readHtmlDocument({ html, pageUrl }, _optional = {}) {
202
199
  const pageHost = siteHost(new URL(pageUrl).hostname);
203
200
  const links = readLinks(raw.anchors, base, pageHost);
204
201
  const images = readImages(raw.images, base);
205
- const description = raw.metas.find(m => attr({ element: m, name: "name" })?.toLowerCase() === "description");
206
- const title = raw.title ? textOf(raw.title) : "";
207
- const lang = raw.html ? attr({ element: raw.html, name: "lang" })?.trim() : undefined;
208
- const descriptionText = description ? collapse(attr({ element: description, name: "content" }) ?? "") : "";
209
202
  return {
210
- ...(title ? { title } : {}),
211
- ...(descriptionText ? { description: descriptionText } : {}),
212
- ...(lang ? { lang } : {}),
203
+ ...headFacts({ raw }),
213
204
  links: links.links, linksTruncated: links.truncated,
214
205
  images: images.images, imagesTruncated: images.truncated,
215
206
  stylesheets: readStylesheets(raw.linkTags, base),
216
207
  meta: readMeta(raw, base),
217
208
  };
218
209
  }
210
+ /** Title, description and lang; empty ones are absent keys. */
211
+ function headFacts({ raw }, _optional = {}) {
212
+ const description = raw.metas.find(m => attr({ element: m, name: "name" })?.toLowerCase() === "description");
213
+ const facts = {
214
+ title: raw.title ? textOf(raw.title) : "",
215
+ description: description ? collapse(attr({ element: description, name: "content" }) ?? "") : "",
216
+ // parse5 always synthesizes the <html> element, so collect() always finds one.
217
+ lang: attr({ element: raw.html, name: "lang" })?.trim(),
218
+ };
219
+ return Object.fromEntries(Object.entries(facts).filter(([, value]) => value));
220
+ }
219
221
  /**
220
222
  * Visible text with one line per block element; scripts, styles, `<head>`, SVG and embeds omitted.
221
223
  * @complexity O(nodes).
@@ -225,28 +227,30 @@ export function htmlToText({ html }, _optional = {}) {
225
227
  const out = [];
226
228
  while (pending.length) {
227
229
  const node = pending.pop();
228
- if (node === "\n") {
229
- out.push("\n");
230
- continue;
231
- }
232
- if (isElement(node) && INVISIBLE.has(node.tagName))
233
- continue;
234
- if (node.nodeName === "#text" && "value" in node) {
235
- out.push(node.value);
236
- continue;
237
- }
238
- if (!("childNodes" in node))
239
- continue;
240
- const block = isElement(node) && BLOCKS.has(node.tagName);
241
- if (block) {
230
+ if (node === "\n")
242
231
  out.push("\n");
243
- pending.push("\n");
244
- }
245
- for (let i = node.childNodes.length - 1; i >= 0; i--)
246
- pending.push(node.childNodes[i]);
232
+ else
233
+ visitText({ node, pending, out });
247
234
  }
248
235
  return out.join("").split("\n").map(line => line.replace(/[^\S\n]+/g, " ").trim()).filter(Boolean).join("\n");
249
236
  }
237
+ /** Emits a text node, or schedules a visible node's children, bracketed by newlines for blocks. */
238
+ function visitText({ node, pending, out }, _optional = {}) {
239
+ if (isElement(node) && INVISIBLE.has(node.tagName))
240
+ return;
241
+ if (node.nodeName === "#text" && "value" in node) {
242
+ out.push(node.value);
243
+ return;
244
+ }
245
+ if (!("childNodes" in node))
246
+ return;
247
+ if (isElement(node) && BLOCKS.has(node.tagName)) {
248
+ out.push("\n");
249
+ pending.push("\n");
250
+ }
251
+ for (let i = node.childNodes.length - 1; i >= 0; i--)
252
+ pending.push(node.childNodes[i]);
253
+ }
250
254
  /**
251
255
  * The page HTML with `<script>`, `<noscript>`, `<template>`, comments, `on*` handler attributes and
252
256
  * `javascript:` attribute values removed (styles and classes kept: theming needs them). Re-serialized by parse5, so it is well-formed rather than byte-identical.