@jini-ai/platform 0.5.9 → 0.6.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -1
- package/dist/asset-cache.js +26 -25
- package/dist/blob-storage.d.ts +7 -2
- package/dist/blob-storage.js +99 -59
- package/dist/child-environment.d.ts +36 -0
- package/dist/child-environment.js +68 -0
- package/dist/download.js +192 -148
- package/dist/fs/atomic-write.js +105 -93
- package/dist/fs/boot-identity.d.ts +16 -0
- package/dist/fs/boot-identity.js +38 -17
- package/dist/fs/directory-home.js +6 -0
- package/dist/fs/directory-registry.d.ts +1 -1
- package/dist/fs/directory-registry.js +23 -16
- package/dist/fs/directory-running-lock.d.ts +1 -1
- package/dist/fs/directory-running-lock.js +18 -7
- package/dist/fs/directory-sync.d.ts +1 -1
- package/dist/fs/directory-sync.js +64 -51
- package/dist/fs/directory-user-config.js +108 -67
- package/dist/fs/directory-workspace.d.ts +11 -0
- package/dist/fs/directory-workspace.js +16 -11
- package/dist/fs/durable-json.js +49 -37
- package/dist/fs/file-lock.js +214 -149
- package/dist/fs/guarded-reader.js +47 -35
- package/dist/fs/user-data-directories.d.ts +1 -1
- package/dist/fs/user-data-directories.js +25 -9
- package/dist/fs/verified-directory-move.d.ts +13 -11
- package/dist/fs/verified-directory-move.js +105 -68
- package/dist/fs/verified-file-relocation.d.ts +1 -1
- package/dist/fs/verified-file-relocation.js +111 -94
- package/dist/http/guarded/client.d.ts +3 -4
- package/dist/http/guarded/client.js +50 -60
- package/dist/http/guarded/fetch-adapter.d.ts +2 -3
- package/dist/http/guarded/fetch-adapter.js +26 -15
- package/dist/mail/builtin-providers.d.ts +39 -0
- package/dist/mail/builtin-providers.js +60 -0
- package/dist/mail/feedback.d.ts +84 -0
- package/dist/mail/feedback.js +90 -0
- package/dist/mail/index.d.ts +11 -0
- package/dist/mail/index.js +6 -0
- package/dist/mail/policy.d.ts +77 -0
- package/dist/mail/policy.js +155 -0
- package/dist/mail/ports.d.ts +56 -7
- package/dist/mail/provider.d.ts +117 -0
- package/dist/mail/provider.js +74 -0
- package/dist/mail/rate-limit.d.ts +33 -0
- package/dist/mail/rate-limit.js +35 -0
- package/dist/mail/sender-identity.d.ts +68 -0
- package/dist/mail/sender-identity.js +50 -0
- package/dist/mail/types.d.ts +9 -0
- package/dist/net/address.d.ts +9 -23
- package/dist/net/address.js +28 -133
- package/dist/net/endpoint-policy.d.ts +26 -2
- package/dist/net/endpoint-policy.js +159 -56
- package/dist/object-store/index.d.ts +2 -0
- package/dist/object-store/index.js +2 -0
- package/dist/object-store/local-fs.d.ts +17 -0
- package/dist/object-store/local-fs.js +146 -0
- package/dist/process.js +2 -0
- package/dist/proxy-env.js +131 -135
- package/dist/secrets/credential-sets/contracts.d.ts +1 -0
- package/dist/secrets/credential-sets/contracts.js +1 -0
- package/dist/secrets/credential-sets/index.d.ts +1 -0
- package/dist/secrets/credential-sets/index.js +1 -0
- package/dist/secrets/credential-sets/provider-matches.d.ts +61 -0
- package/dist/secrets/credential-sets/provider-matches.js +61 -0
- package/dist/secrets/credential-sets/types.d.ts +7 -0
- package/dist/secrets/credential-sets/vault.meta.js +58 -20
- package/dist/secrets/credential-sets/vault.repo.memory.d.ts +7 -0
- package/dist/secrets/credential-sets/vault.repo.memory.js +43 -36
- package/dist/secrets/credential-sets/vault.service.d.ts +9 -0
- package/dist/secrets/credential-sets/vault.service.js +131 -61
- package/dist/secrets/credential-sets/vault.types.d.ts +11 -3
- package/dist/secrets/credential-sets/vault.types.js +22 -8
- package/dist/secrets/keyring.env.js +14 -10
- package/dist/toolchain.js +119 -91
- package/dist/web/html-document.js +73 -69
- package/dist/web/html-to-markdown.js +127 -102
- package/dist/web/web-page.js +160 -80
- package/package.json +12 -4
|
@@ -24,15 +24,7 @@ export class EnvOrFileKeyring {
|
|
|
24
24
|
hkdfSalt;
|
|
25
25
|
cachedRootKey;
|
|
26
26
|
constructor(required, optional = {}) {
|
|
27
|
-
|
|
28
|
-
requireHostOption(required?.keyFilePath, "keyFilePath");
|
|
29
|
-
requireHostOption(required?.hkdfSalt, "hkdfSalt");
|
|
30
|
-
if (typeof required.env?.read !== "function")
|
|
31
|
-
throw new TypeError("env.read is required");
|
|
32
|
-
if (typeof required.allowFileFallback !== "boolean")
|
|
33
|
-
throw new TypeError("allowFileFallback must be an explicit boolean");
|
|
34
|
-
if (typeof required.allowFileAutoGenerate !== "boolean")
|
|
35
|
-
throw new TypeError("allowFileAutoGenerate must be an explicit boolean");
|
|
27
|
+
assertEnvOrFileKeyringRequired({ required });
|
|
36
28
|
this.messages = optional.messages ?? defaultPlatformMessages;
|
|
37
29
|
this.env = required.env;
|
|
38
30
|
this.hkdfSalt = required.hkdfSalt;
|
|
@@ -178,7 +170,7 @@ function unusableRootKeyMessage(input) {
|
|
|
178
170
|
const refusal = `${input.subject} is not usable as a root key: ${input.detail}. The keyring refuses to derive any key material from it rather than sealing under a shortened or empty key.`;
|
|
179
171
|
if (input.sealedWarningSubject === undefined)
|
|
180
172
|
return refusal;
|
|
181
|
-
return (`${refusal} ${
|
|
173
|
+
return (`${refusal} ${input.messages.rootKeySealedWarning({ subject: input.sealedWarningSubject })}` +
|
|
182
174
|
`not from a real ${ROOT_KEY_LENGTH_BYTES}-byte key, so those stored credentials are not protected as intended — with empty or very short material the derived key is computable from published constants. ` +
|
|
183
175
|
`Replacing or regenerating the key will NOT restore them; it will make them unreadable. ` +
|
|
184
176
|
`Do not overwrite, delete or regenerate this key until you have decided what to do with the credentials already stored.`);
|
|
@@ -264,6 +256,18 @@ function isAlreadyExistsError(err) {
|
|
|
264
256
|
const code = err.code;
|
|
265
257
|
return code === "EEXIST" || code === "ELOOP";
|
|
266
258
|
}
|
|
259
|
+
/** Every source permission is explicit; checked in order so the first missing option is named. */
|
|
260
|
+
function assertEnvOrFileKeyringRequired({ required }) {
|
|
261
|
+
requireHostOption(required?.envVarName, "envVarName");
|
|
262
|
+
requireHostOption(required?.keyFilePath, "keyFilePath");
|
|
263
|
+
requireHostOption(required?.hkdfSalt, "hkdfSalt");
|
|
264
|
+
if (typeof required.env?.read !== "function")
|
|
265
|
+
throw new TypeError("env.read is required");
|
|
266
|
+
if (typeof required.allowFileFallback !== "boolean")
|
|
267
|
+
throw new TypeError("allowFileFallback must be an explicit boolean");
|
|
268
|
+
if (typeof required.allowFileAutoGenerate !== "boolean")
|
|
269
|
+
throw new TypeError("allowFileAutoGenerate must be an explicit boolean");
|
|
270
|
+
}
|
|
267
271
|
function requireHostOption(value, name) {
|
|
268
272
|
if (value === undefined || value === null)
|
|
269
273
|
throw new TypeError(`${name} is required`);
|
package/dist/toolchain.js
CHANGED
|
@@ -25,6 +25,14 @@ function resolveUserScopedHome(raw, home) {
|
|
|
25
25
|
}
|
|
26
26
|
return isAbsolute(value) ? value : null;
|
|
27
27
|
}
|
|
28
|
+
/** @internal Trim an optional env value, treating a missing/non-string value as blank. */
|
|
29
|
+
function trimmedEnv(value) {
|
|
30
|
+
return typeof value === "string" ? value.trim() : "";
|
|
31
|
+
}
|
|
32
|
+
/** @internal Whether the current process runs on Windows (read live so tests can swap `process.platform`). */
|
|
33
|
+
function isWindows() {
|
|
34
|
+
return process.platform === "win32";
|
|
35
|
+
}
|
|
28
36
|
// Single source of truth for "user-level CLI install locations the daemon
|
|
29
37
|
// must search even when launched with a minimal PATH". GUI launchers
|
|
30
38
|
// (macOS .app bundles, Linux .desktop files) typically inherit a stripped
|
|
@@ -46,7 +54,7 @@ function resolveUserScopedHome(raw, home) {
|
|
|
46
54
|
*/
|
|
47
55
|
export function wellKnownUserToolchainBins(options = {}) {
|
|
48
56
|
const home = options.home ?? homedir();
|
|
49
|
-
const includeSystemBins = options.includeSystemBins ??
|
|
57
|
+
const includeSystemBins = options.includeSystemBins ?? !isWindows();
|
|
50
58
|
const env = options.env ?? process.env;
|
|
51
59
|
const dirs = [];
|
|
52
60
|
// Vite+ global installs expose CLI shims from VP_HOME/bin (default
|
|
@@ -57,6 +65,40 @@ export function wellKnownUserToolchainBins(options = {}) {
|
|
|
57
65
|
if (vpHome) {
|
|
58
66
|
dirs.push(join(vpHome, "bin"));
|
|
59
67
|
}
|
|
68
|
+
dirs.push(...npmPrefixBins({ env, home }));
|
|
69
|
+
dirs.push(...conventionalUserBins({ home }));
|
|
70
|
+
dirs.push(...windowsUserBins({ env, home }));
|
|
71
|
+
// Mise shims: makes every tool installed with `mise install` visible to
|
|
72
|
+
// GUI-launched daemons even when the process inherits a stripped PATH.
|
|
73
|
+
// Respect MISE_DATA_DIR (the official way to relocate the whole mise tree).
|
|
74
|
+
// We only fall back to the legacy ~/.mise/shims path when no explicit
|
|
75
|
+
// MISE_DATA_DIR override is provided.
|
|
76
|
+
const miseDataOverride = resolveUserScopedHome(env.MISE_DATA_DIR, home);
|
|
77
|
+
const miseData = miseDataOverride ?? join(home, ".local", "share", "mise");
|
|
78
|
+
dirs.push(join(miseData, "shims"));
|
|
79
|
+
if (!miseDataOverride) {
|
|
80
|
+
dirs.push(join(home, ".mise", "shims"));
|
|
81
|
+
}
|
|
82
|
+
if (includeSystemBins) {
|
|
83
|
+
dirs.push("/opt/homebrew/bin", "/usr/local/bin");
|
|
84
|
+
}
|
|
85
|
+
// Per-version Node toolchains: scan the install root and surface every
|
|
86
|
+
// version directory's bin folder. Best-effort — missing roots simply
|
|
87
|
+
// contribute nothing.
|
|
88
|
+
// When MISE_DATA_DIR is set we use the same root for consistency with shims.
|
|
89
|
+
const miseInstalls = join(miseData, "installs");
|
|
90
|
+
// Surface bin dirs for npm-packaged toolchains installed under the mise
|
|
91
|
+
// installs root (e.g. `npm-openai-codex`).
|
|
92
|
+
dirs.push(...existingChildBinDirs(join(miseInstalls, "npm-openai-codex"), ["bin"]));
|
|
93
|
+
for (const installRoot of nodeInstallRoots({ env, home, miseInstalls })) {
|
|
94
|
+
dirs.push(...existingChildBinDirs(installRoot.root, installRoot.segments));
|
|
95
|
+
}
|
|
96
|
+
return dirs;
|
|
97
|
+
}
|
|
98
|
+
/** @internal The npm-prefix bins derived from the env plus the Windows `%APPDATA%\npm` default. */
|
|
99
|
+
function npmPrefixBins(input) {
|
|
100
|
+
const { env, home } = input;
|
|
101
|
+
const dirs = [];
|
|
60
102
|
// The user's *explicit* npm prefix outranks every conventional
|
|
61
103
|
// location below — including `~/.local/bin`. The env var is the
|
|
62
104
|
// user's current npm configuration, so a binary installed via
|
|
@@ -72,21 +114,18 @@ export function wellKnownUserToolchainBins(options = {}) {
|
|
|
72
114
|
// Trim before length-checking so accidental whitespace-only values
|
|
73
115
|
// (`NPM_CONFIG_PREFIX=" "`) do not produce a `/bin`-suffixed garbage
|
|
74
116
|
// entry.
|
|
75
|
-
const
|
|
76
|
-
if (
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
if (process.platform === "win32") {
|
|
88
|
-
dirs.push(npmPrefix);
|
|
89
|
-
}
|
|
117
|
+
const npmPrefix = trimmedEnv(env.NPM_CONFIG_PREFIX ?? env.npm_config_prefix);
|
|
118
|
+
if (npmPrefix.length > 0) {
|
|
119
|
+
// Unix: npm global binaries live in <prefix>/bin. Always add it as a
|
|
120
|
+
// search-path candidate — existence is irrelevant for a PATH directory,
|
|
121
|
+
// and this preserves the long-standing helper contract.
|
|
122
|
+
dirs.push(join(npmPrefix, "bin"));
|
|
123
|
+
// Windows: npm installs global binaries directly into the prefix root
|
|
124
|
+
// (there is no /bin subdirectory), so add the root as well. Additive —
|
|
125
|
+
// the <prefix>/bin entry above is left untouched for cross-platform
|
|
126
|
+
// parity.
|
|
127
|
+
if (isWindows()) {
|
|
128
|
+
dirs.push(npmPrefix);
|
|
90
129
|
}
|
|
91
130
|
}
|
|
92
131
|
// Windows: %APPDATA%\npm is npm's default global prefix. NPM_CONFIG_PREFIX /
|
|
@@ -94,53 +133,59 @@ export function wellKnownUserToolchainBins(options = {}) {
|
|
|
94
133
|
// child-process environments, so the block above no-ops for most Windows
|
|
95
134
|
// users. Add the default location unconditionally — consistent with the
|
|
96
135
|
// conventional dirs below, which are also added without an existence check.
|
|
97
|
-
if (
|
|
136
|
+
if (isWindows()) {
|
|
98
137
|
dirs.push(join(home, "AppData", "Roaming", "npm"));
|
|
99
138
|
}
|
|
100
|
-
dirs
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
139
|
+
return dirs;
|
|
140
|
+
}
|
|
141
|
+
/** @internal Conventional home-relative toolchain bin dirs, added without an existence check. */
|
|
142
|
+
function conventionalUserBins(input) {
|
|
143
|
+
const { home } = input;
|
|
144
|
+
return [
|
|
145
|
+
join(home, ".local", "bin"),
|
|
146
|
+
join(home, ".vite-plus", "bin"),
|
|
147
|
+
join(home, ".kimi-code", "bin"),
|
|
148
|
+
join(home, ".opencode", "bin"),
|
|
149
|
+
join(home, ".bun", "bin"),
|
|
150
|
+
join(home, ".volta", "bin"),
|
|
151
|
+
join(home, ".asdf", "shims"),
|
|
152
|
+
join(home, "Library", "pnpm"),
|
|
153
|
+
join(home, ".cargo", "bin"),
|
|
154
|
+
// Common user-level npm prefixes for sudo-free global installs.
|
|
155
|
+
// ~/.npm-global is the dominant non-canonical convention shipped
|
|
156
|
+
// in most third-party "fix npm EACCES" tutorials, and
|
|
157
|
+
// ~/.npm-packages is the second-most common variant. Without
|
|
158
|
+
// these, GUI-launched daemons miss `npm i -g`'d CLIs even though
|
|
159
|
+
// they resolve cleanly from the user's shell.
|
|
160
|
+
join(home, ".npm-global", "bin"),
|
|
161
|
+
join(home, ".npm-packages", "bin"),
|
|
162
|
+
// Other common user-level toolchains that install CLI shims outside the
|
|
163
|
+
// Node ecosystem but still ship agent CLIs (or their dependencies):
|
|
164
|
+
// Deno's install root, Go's default GOBIN, and pyenv's shim dir. All are
|
|
165
|
+
// best-effort — a missing dir contributes nothing.
|
|
166
|
+
join(home, ".deno", "bin"),
|
|
167
|
+
join(home, "go", "bin"),
|
|
168
|
+
join(home, ".pyenv", "shims"),
|
|
169
|
+
];
|
|
170
|
+
}
|
|
171
|
+
/** @internal Windows-only scoop shims and `%APPDATA%\npm`; empty on POSIX. */
|
|
172
|
+
function windowsUserBins(input) {
|
|
113
173
|
// Windows-only user install roots that GUI launches miss. Scoop drops
|
|
114
174
|
// shims under ~/scoop/shims, and npm's global prefix on Windows defaults
|
|
115
175
|
// to %APPDATA%\npm rather than a `bin` subdir.
|
|
116
|
-
if (
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
176
|
+
if (!isWindows())
|
|
177
|
+
return [];
|
|
178
|
+
const dirs = [join(input.home, "scoop", "shims")];
|
|
179
|
+
const appData = trimmedEnv(input.env.APPDATA);
|
|
180
|
+
if (appData.length > 0) {
|
|
181
|
+
dirs.push(join(appData, "npm"));
|
|
122
182
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
const
|
|
129
|
-
const miseData = miseDataOverride || join(home, ".local", "share", "mise");
|
|
130
|
-
dirs.push(join(miseData, "shims"));
|
|
131
|
-
if (!miseDataOverride) {
|
|
132
|
-
dirs.push(join(home, ".mise", "shims"));
|
|
133
|
-
}
|
|
134
|
-
if (includeSystemBins) {
|
|
135
|
-
dirs.push("/opt/homebrew/bin", "/usr/local/bin");
|
|
136
|
-
}
|
|
137
|
-
// Per-version Node toolchains: scan the install root and surface every
|
|
138
|
-
// version directory's bin folder. Best-effort — missing roots simply
|
|
139
|
-
// contribute nothing.
|
|
140
|
-
// When MISE_DATA_DIR is set we use the same root for consistency with shims.
|
|
141
|
-
const miseInstalls = join(miseData, "installs");
|
|
142
|
-
dirs.push(...existingMiseNpmPackageBinDirs(miseInstalls));
|
|
143
|
-
const nodeInstallRoots = [
|
|
183
|
+
return dirs;
|
|
184
|
+
}
|
|
185
|
+
/** @internal Per-version Node install roots to probe, in priority order (mise, nvm, fnm, then Windows fnm). */
|
|
186
|
+
function nodeInstallRoots(input) {
|
|
187
|
+
const { env, home, miseInstalls } = input;
|
|
188
|
+
const roots = [
|
|
144
189
|
{
|
|
145
190
|
root: join(miseInstalls, "node"),
|
|
146
191
|
segments: ["bin"],
|
|
@@ -166,38 +211,23 @@ export function wellKnownUserToolchainBins(options = {}) {
|
|
|
166
211
|
// inherits a stripped PATH and reads no shell rc, so without an explicit
|
|
167
212
|
// probe fnm-managed Node — and every agent CLI it runs — is silently
|
|
168
213
|
// undetected on Windows. See issues #3517 and #3062.
|
|
169
|
-
if (
|
|
170
|
-
const
|
|
171
|
-
|
|
172
|
-
if (fnmDirOverride.length > 0) {
|
|
173
|
-
fnmRoots.push(fnmDirOverride);
|
|
174
|
-
}
|
|
175
|
-
else {
|
|
176
|
-
for (const base of [env.LOCALAPPDATA, env.APPDATA]) {
|
|
177
|
-
const trimmed = typeof base === "string" ? base.trim() : "";
|
|
178
|
-
if (trimmed.length > 0)
|
|
179
|
-
fnmRoots.push(join(trimmed, "fnm"));
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
for (const fnmRoot of fnmRoots) {
|
|
183
|
-
nodeInstallRoots.push({ root: join(fnmRoot, "node-versions"), segments: ["installation"] });
|
|
214
|
+
if (isWindows()) {
|
|
215
|
+
for (const fnmRoot of windowsFnmRoots({ env })) {
|
|
216
|
+
roots.push({ root: join(fnmRoot, "node-versions"), segments: ["installation"] });
|
|
184
217
|
}
|
|
185
218
|
}
|
|
186
|
-
|
|
187
|
-
for (const dir of existingChildBinDirs(installRoot.root, installRoot.segments)) {
|
|
188
|
-
dirs.push(dir);
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
|
-
return dirs;
|
|
219
|
+
return roots;
|
|
192
220
|
}
|
|
193
|
-
/** @internal
|
|
194
|
-
function
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
221
|
+
/** @internal Windows fnm roots: `FNM_DIR` when set, else `%LOCALAPPDATA%\fnm` then `%APPDATA%\fnm`. */
|
|
222
|
+
function windowsFnmRoots(input) {
|
|
223
|
+
const { env } = input;
|
|
224
|
+
const fnmDirOverride = trimmedEnv(env.FNM_DIR);
|
|
225
|
+
if (fnmDirOverride.length > 0)
|
|
226
|
+
return [fnmDirOverride];
|
|
227
|
+
return [env.LOCALAPPDATA, env.APPDATA]
|
|
228
|
+
.map(trimmedEnv)
|
|
229
|
+
.filter((base) => base.length > 0)
|
|
230
|
+
.map((base) => join(base, "fnm"));
|
|
201
231
|
}
|
|
202
232
|
/** @internal List existing `<root>/<versionDir>/<...segments>` directories, version-sorted, skipping non-directories. */
|
|
203
233
|
function existingChildBinDirs(root, segments) {
|
|
@@ -209,7 +239,9 @@ function existingChildBinDirs(root, segments) {
|
|
|
209
239
|
catch {
|
|
210
240
|
return out;
|
|
211
241
|
}
|
|
212
|
-
|
|
242
|
+
// Sort directory entries by semver-like name (newest first), falling back to locale order.
|
|
243
|
+
const sorted = [...entries].sort((left, right) => compareVersionLikeDirNames(left.name, right.name));
|
|
244
|
+
for (const entry of sorted) {
|
|
213
245
|
if (!entry.isDirectory() && !entry.isSymbolicLink())
|
|
214
246
|
continue;
|
|
215
247
|
const candidate = join(root, entry.name, ...segments);
|
|
@@ -218,10 +250,6 @@ function existingChildBinDirs(root, segments) {
|
|
|
218
250
|
}
|
|
219
251
|
return out;
|
|
220
252
|
}
|
|
221
|
-
/** @internal Sort directory entries by semver-like name (newest first), falling back to locale order. */
|
|
222
|
-
function sortVersionedDirEntries(entries) {
|
|
223
|
-
return [...entries].sort((left, right) => compareVersionLikeDirNames(left.name, right.name));
|
|
224
|
-
}
|
|
225
253
|
/** @internal Compare two version-like directory names descending; semver-parseable names precede non-parseable ones. */
|
|
226
254
|
function compareVersionLikeDirNames(left, right) {
|
|
227
255
|
const leftSemver = parseVersionLikeDirName(left);
|
|
@@ -83,65 +83,63 @@ export function imageSource({ img }, _optional = {}) {
|
|
|
83
83
|
return src;
|
|
84
84
|
return attr({ element: img, name: "data-src" }) ?? attr({ element: img, name: "data-lazy-src" }) ?? attr({ element: img, name: "srcset" })?.trim().split(/\s+/)[0];
|
|
85
85
|
}
|
|
86
|
+
/** A Map, not an object literal, so tag names such as `constructor` never hit Object.prototype. */
|
|
87
|
+
const COLLECTED = new Map([["a", "anchors"], ["img", "images"], ["link", "linkTags"], ["meta", "metas"]]);
|
|
86
88
|
function collect(document) {
|
|
87
89
|
const raw = { anchors: [], images: [], linkTags: [], metas: [] };
|
|
88
90
|
for (const node of walk(document)) {
|
|
89
91
|
if (!isElement(node))
|
|
90
92
|
continue;
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
raw.linkTags.push(node);
|
|
100
|
-
break;
|
|
101
|
-
case "meta":
|
|
102
|
-
raw.metas.push(node);
|
|
103
|
-
break;
|
|
104
|
-
case "title":
|
|
105
|
-
raw.title ??= node;
|
|
106
|
-
break;
|
|
107
|
-
case "html":
|
|
108
|
-
raw.html ??= node;
|
|
109
|
-
break;
|
|
110
|
-
case "base":
|
|
111
|
-
raw.base ??= attr({ element: node, name: "href" });
|
|
112
|
-
break;
|
|
113
|
-
}
|
|
93
|
+
const list = COLLECTED.get(node.tagName);
|
|
94
|
+
if (list)
|
|
95
|
+
raw[list].push(node);
|
|
96
|
+
else if (node.tagName === "title" || node.tagName === "html")
|
|
97
|
+
raw[node.tagName] ??= node;
|
|
98
|
+
// The first base WITH an href wins, as in browsers.
|
|
99
|
+
else if (node.tagName === "base")
|
|
100
|
+
raw.base ??= attr({ element: node, name: "href" });
|
|
114
101
|
}
|
|
115
102
|
return raw;
|
|
116
103
|
}
|
|
104
|
+
/** Meta keys whose values are URLs and are returned absolute. */
|
|
105
|
+
const URL_META = new Set(["og:image", "og:url", "twitter:image"]);
|
|
106
|
+
/** `<link rel>` tokens copied into meta, in emission order, under their meta key. */
|
|
107
|
+
const LINK_META = [["canonical", "canonical"], ["icon", "favicon"], ["apple-touch-icon", "apple-touch-icon"]];
|
|
117
108
|
function readMeta(raw, base) {
|
|
118
109
|
const meta = {};
|
|
119
|
-
const put = (key, value) => {
|
|
120
|
-
if (value === undefined || key in meta || Object.keys(meta).length >= MAX_META)
|
|
121
|
-
return;
|
|
122
|
-
meta[key] = collapse(value).slice(0, MAX_META_VALUE);
|
|
123
|
-
};
|
|
124
110
|
for (const element of raw.metas) {
|
|
125
|
-
const
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
if (property?.startsWith("og:"))
|
|
129
|
-
put(property, property === "og:image" || property === "og:url" ? absolute({ raw: content, base, protocols: ASSET_PROTOCOLS }) : content);
|
|
130
|
-
else if (name && META_NAMES.has(name))
|
|
131
|
-
put(name, name === "twitter:image" ? absolute({ raw: content, base, protocols: ASSET_PROTOCOLS }) : content);
|
|
111
|
+
const entry = metaTagEntry({ element, base });
|
|
112
|
+
if (entry)
|
|
113
|
+
putMeta({ meta, key: entry[0], value: entry[1] });
|
|
132
114
|
}
|
|
133
115
|
for (const element of raw.linkTags) {
|
|
134
|
-
const
|
|
116
|
+
const rels = relTokens({ element });
|
|
135
117
|
const href = absolute({ raw: attr({ element, name: "href" }), base, protocols: ASSET_PROTOCOLS });
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
put("favicon", href);
|
|
140
|
-
if (rel.includes("apple-touch-icon"))
|
|
141
|
-
put("apple-touch-icon", href);
|
|
118
|
+
for (const [rel, key] of LINK_META)
|
|
119
|
+
if (rels.includes(rel))
|
|
120
|
+
putMeta({ meta, key, value: href });
|
|
142
121
|
}
|
|
143
122
|
return meta;
|
|
144
123
|
}
|
|
124
|
+
/** First value per key wins; the entry count and each value are capped. */
|
|
125
|
+
function putMeta({ meta, key, value }, _optional = {}) {
|
|
126
|
+
if (value === undefined || key in meta || Object.keys(meta).length >= MAX_META)
|
|
127
|
+
return;
|
|
128
|
+
meta[key] = collapse(value).slice(0, MAX_META_VALUE);
|
|
129
|
+
}
|
|
130
|
+
/** Every `og:*` property, or an allowlisted `name`; URL-valued keys are resolved. */
|
|
131
|
+
function metaTagEntry({ element, base }, _optional = {}) {
|
|
132
|
+
const property = attr({ element, name: "property" })?.toLowerCase();
|
|
133
|
+
const name = attr({ element, name: "name" })?.toLowerCase();
|
|
134
|
+
const key = property?.startsWith("og:") ? property : name && META_NAMES.has(name) ? name : undefined;
|
|
135
|
+
if (key === undefined)
|
|
136
|
+
return undefined;
|
|
137
|
+
const content = attr({ element, name: "content" });
|
|
138
|
+
return [key, URL_META.has(key) ? absolute({ raw: content, base, protocols: ASSET_PROTOCOLS }) : content];
|
|
139
|
+
}
|
|
140
|
+
function relTokens({ element }, _optional = {}) {
|
|
141
|
+
return (attr({ element, name: "rel" }) ?? "").toLowerCase().split(/\s+/);
|
|
142
|
+
}
|
|
145
143
|
function readLinks(anchors, base, pageHost) {
|
|
146
144
|
const seen = new Set();
|
|
147
145
|
const links = [];
|
|
@@ -179,9 +177,8 @@ function readImages(images, base) {
|
|
|
179
177
|
function readStylesheets(linkTags, base) {
|
|
180
178
|
const out = new Set();
|
|
181
179
|
for (const element of linkTags) {
|
|
182
|
-
const rel = (attr({ element, name: "rel" }) ?? "").toLowerCase().split(/\s+/);
|
|
183
180
|
const href = absolute({ raw: attr({ element, name: "href" }), base, protocols: ASSET_PROTOCOLS });
|
|
184
|
-
if (
|
|
181
|
+
if (relTokens({ element }).includes("stylesheet") && href && out.size < MAX_STYLESHEETS)
|
|
185
182
|
out.add(href);
|
|
186
183
|
}
|
|
187
184
|
return [...out];
|
|
@@ -202,20 +199,25 @@ export function readHtmlDocument({ html, pageUrl }, _optional = {}) {
|
|
|
202
199
|
const pageHost = siteHost(new URL(pageUrl).hostname);
|
|
203
200
|
const links = readLinks(raw.anchors, base, pageHost);
|
|
204
201
|
const images = readImages(raw.images, base);
|
|
205
|
-
const description = raw.metas.find(m => attr({ element: m, name: "name" })?.toLowerCase() === "description");
|
|
206
|
-
const title = raw.title ? textOf(raw.title) : "";
|
|
207
|
-
const lang = raw.html ? attr({ element: raw.html, name: "lang" })?.trim() : undefined;
|
|
208
|
-
const descriptionText = description ? collapse(attr({ element: description, name: "content" }) ?? "") : "";
|
|
209
202
|
return {
|
|
210
|
-
...(
|
|
211
|
-
...(descriptionText ? { description: descriptionText } : {}),
|
|
212
|
-
...(lang ? { lang } : {}),
|
|
203
|
+
...headFacts({ raw }),
|
|
213
204
|
links: links.links, linksTruncated: links.truncated,
|
|
214
205
|
images: images.images, imagesTruncated: images.truncated,
|
|
215
206
|
stylesheets: readStylesheets(raw.linkTags, base),
|
|
216
207
|
meta: readMeta(raw, base),
|
|
217
208
|
};
|
|
218
209
|
}
|
|
210
|
+
/** Title, description and lang; empty ones are absent keys. */
|
|
211
|
+
function headFacts({ raw }, _optional = {}) {
|
|
212
|
+
const description = raw.metas.find(m => attr({ element: m, name: "name" })?.toLowerCase() === "description");
|
|
213
|
+
const facts = {
|
|
214
|
+
title: raw.title ? textOf(raw.title) : "",
|
|
215
|
+
description: description ? collapse(attr({ element: description, name: "content" }) ?? "") : "",
|
|
216
|
+
// parse5 always synthesizes the <html> element, so collect() always finds one.
|
|
217
|
+
lang: attr({ element: raw.html, name: "lang" })?.trim(),
|
|
218
|
+
};
|
|
219
|
+
return Object.fromEntries(Object.entries(facts).filter(([, value]) => value));
|
|
220
|
+
}
|
|
219
221
|
/**
|
|
220
222
|
* Visible text with one line per block element; scripts, styles, `<head>`, SVG and embeds omitted.
|
|
221
223
|
* @complexity O(nodes).
|
|
@@ -225,28 +227,30 @@ export function htmlToText({ html }, _optional = {}) {
|
|
|
225
227
|
const out = [];
|
|
226
228
|
while (pending.length) {
|
|
227
229
|
const node = pending.pop();
|
|
228
|
-
if (node === "\n")
|
|
229
|
-
out.push("\n");
|
|
230
|
-
continue;
|
|
231
|
-
}
|
|
232
|
-
if (isElement(node) && INVISIBLE.has(node.tagName))
|
|
233
|
-
continue;
|
|
234
|
-
if (node.nodeName === "#text" && "value" in node) {
|
|
235
|
-
out.push(node.value);
|
|
236
|
-
continue;
|
|
237
|
-
}
|
|
238
|
-
if (!("childNodes" in node))
|
|
239
|
-
continue;
|
|
240
|
-
const block = isElement(node) && BLOCKS.has(node.tagName);
|
|
241
|
-
if (block) {
|
|
230
|
+
if (node === "\n")
|
|
242
231
|
out.push("\n");
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
for (let i = node.childNodes.length - 1; i >= 0; i--)
|
|
246
|
-
pending.push(node.childNodes[i]);
|
|
232
|
+
else
|
|
233
|
+
visitText({ node, pending, out });
|
|
247
234
|
}
|
|
248
235
|
return out.join("").split("\n").map(line => line.replace(/[^\S\n]+/g, " ").trim()).filter(Boolean).join("\n");
|
|
249
236
|
}
|
|
237
|
+
/** Emits a text node, or schedules a visible node's children, bracketed by newlines for blocks. */
|
|
238
|
+
function visitText({ node, pending, out }, _optional = {}) {
|
|
239
|
+
if (isElement(node) && INVISIBLE.has(node.tagName))
|
|
240
|
+
return;
|
|
241
|
+
if (node.nodeName === "#text" && "value" in node) {
|
|
242
|
+
out.push(node.value);
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
if (!("childNodes" in node))
|
|
246
|
+
return;
|
|
247
|
+
if (isElement(node) && BLOCKS.has(node.tagName)) {
|
|
248
|
+
out.push("\n");
|
|
249
|
+
pending.push("\n");
|
|
250
|
+
}
|
|
251
|
+
for (let i = node.childNodes.length - 1; i >= 0; i--)
|
|
252
|
+
pending.push(node.childNodes[i]);
|
|
253
|
+
}
|
|
250
254
|
/**
|
|
251
255
|
* The page HTML with `<script>`, `<noscript>`, `<template>`, comments, `on*` handler attributes and
|
|
252
256
|
* `javascript:` attribute values removed (styles and classes kept: theming needs them). Re-serialized by parse5, so it is well-formed rather than byte-identical.
|