@de-otio/chaoskb-client 0.3.6 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/bootstrap.d.ts +11 -3
- package/dist/cli/bootstrap.d.ts.map +1 -1
- package/dist/cli/bootstrap.js +181 -126
- package/dist/cli/bootstrap.js.map +1 -1
- package/dist/cli/commands/config.d.ts +7 -4
- package/dist/cli/commands/config.d.ts.map +1 -1
- package/dist/cli/commands/config.js +161 -134
- package/dist/cli/commands/config.js.map +1 -1
- package/dist/cli/commands/devices.d.ts.map +1 -1
- package/dist/cli/commands/devices.js +58 -33
- package/dist/cli/commands/devices.js.map +1 -1
- package/dist/cli/commands/export.d.ts.map +1 -1
- package/dist/cli/commands/export.js +6 -9
- package/dist/cli/commands/export.js.map +1 -1
- package/dist/cli/commands/import.js +1 -1
- package/dist/cli/commands/import.js.map +1 -1
- package/dist/cli/commands/projects.d.ts.map +1 -1
- package/dist/cli/commands/projects.js +33 -10
- package/dist/cli/commands/projects.js.map +1 -1
- package/dist/cli/commands/rotate-key.d.ts +3 -3
- package/dist/cli/commands/rotate-key.d.ts.map +1 -1
- package/dist/cli/commands/rotate-key.js +88 -35
- package/dist/cli/commands/rotate-key.js.map +1 -1
- package/dist/cli/commands/setup-sync.d.ts.map +1 -1
- package/dist/cli/commands/setup-sync.js +22 -4
- package/dist/cli/commands/setup-sync.js.map +1 -1
- package/dist/cli/mcp-server.d.ts.map +1 -1
- package/dist/cli/mcp-server.js +90 -42
- package/dist/cli/mcp-server.js.map +1 -1
- package/dist/crypto/aad.d.ts +2 -5
- package/dist/crypto/aad.d.ts.map +1 -1
- package/dist/crypto/aad.js +2 -8
- package/dist/crypto/aad.js.map +1 -1
- package/dist/crypto/aead.d.ts +8 -16
- package/dist/crypto/aead.d.ts.map +1 -1
- package/dist/crypto/aead.js +10 -36
- package/dist/crypto/aead.js.map +1 -1
- package/dist/crypto/blob-id.d.ts +2 -3
- package/dist/crypto/blob-id.d.ts.map +1 -1
- package/dist/crypto/blob-id.js +2 -30
- package/dist/crypto/blob-id.js.map +1 -1
- package/dist/crypto/canonical-json.d.ts +5 -3
- package/dist/crypto/canonical-json.d.ts.map +1 -1
- package/dist/crypto/canonical-json.js +5 -85
- package/dist/crypto/canonical-json.js.map +1 -1
- package/dist/crypto/commitment.d.ts +3 -9
- package/dist/crypto/commitment.d.ts.map +1 -1
- package/dist/crypto/commitment.js +3 -27
- package/dist/crypto/commitment.js.map +1 -1
- package/dist/crypto/encryption-service.d.ts +3 -0
- package/dist/crypto/encryption-service.d.ts.map +1 -1
- package/dist/crypto/encryption-service.js +10 -6
- package/dist/crypto/encryption-service.js.map +1 -1
- package/dist/crypto/envelope-cbor.d.ts +4 -34
- package/dist/crypto/envelope-cbor.d.ts.map +1 -1
- package/dist/crypto/envelope-cbor.js +4 -121
- package/dist/crypto/envelope-cbor.js.map +1 -1
- package/dist/crypto/envelope.d.ts +1 -31
- package/dist/crypto/envelope.d.ts.map +1 -1
- package/dist/crypto/envelope.js +31 -137
- package/dist/crypto/envelope.js.map +1 -1
- package/dist/crypto/hkdf.d.ts +7 -11
- package/dist/crypto/hkdf.d.ts.map +1 -1
- package/dist/crypto/hkdf.js +9 -18
- package/dist/crypto/hkdf.js.map +1 -1
- package/dist/crypto/index.d.ts +9 -4
- package/dist/crypto/index.d.ts.map +1 -1
- package/dist/crypto/index.js +9 -4
- package/dist/crypto/index.js.map +1 -1
- package/dist/crypto/ssh-keys.d.ts +17 -10
- package/dist/crypto/ssh-keys.d.ts.map +1 -1
- package/dist/crypto/ssh-keys.js +28 -108
- package/dist/crypto/ssh-keys.js.map +1 -1
- package/dist/crypto/types.d.ts +18 -88
- package/dist/crypto/types.d.ts.map +1 -1
- package/dist/crypto/types.js +3 -0
- package/dist/crypto/types.js.map +1 -1
- package/dist/pipeline/content-pipeline.d.ts.map +1 -1
- package/dist/pipeline/content-pipeline.js +19 -3
- package/dist/pipeline/content-pipeline.js.map +1 -1
- package/dist/pipeline/extract.d.ts +8 -0
- package/dist/pipeline/extract.d.ts.map +1 -1
- package/dist/pipeline/extract.js +15 -4
- package/dist/pipeline/extract.js.map +1 -1
- package/dist/pipeline/fetch-browser.d.ts +29 -0
- package/dist/pipeline/fetch-browser.d.ts.map +1 -0
- package/dist/pipeline/fetch-browser.js +98 -0
- package/dist/pipeline/fetch-browser.js.map +1 -0
- package/package.json +4 -1
- package/dist/crypto/argon2.d.ts +0 -11
- package/dist/crypto/argon2.d.ts.map +0 -1
- package/dist/crypto/argon2.js +0 -33
- package/dist/crypto/argon2.js.map +0 -1
- package/dist/crypto/invite.d.ts +0 -31
- package/dist/crypto/invite.d.ts.map +0 -1
- package/dist/crypto/invite.js +0 -139
- package/dist/crypto/invite.js.map +0 -1
- package/dist/crypto/keyring.d.ts +0 -37
- package/dist/crypto/keyring.d.ts.map +0 -1
- package/dist/crypto/keyring.js +0 -219
- package/dist/crypto/keyring.js.map +0 -1
- package/dist/crypto/known-keys.d.ts +0 -34
- package/dist/crypto/known-keys.d.ts.map +0 -1
- package/dist/crypto/known-keys.js +0 -114
- package/dist/crypto/known-keys.js.map +0 -1
- package/dist/crypto/project-keys.d.ts +0 -26
- package/dist/crypto/project-keys.d.ts.map +0 -1
- package/dist/crypto/project-keys.js +0 -69
- package/dist/crypto/project-keys.js.map +0 -1
- package/dist/crypto/secure-buffer.d.ts +0 -31
- package/dist/crypto/secure-buffer.d.ts.map +0 -1
- package/dist/crypto/secure-buffer.js +0 -61
- package/dist/crypto/secure-buffer.js.map +0 -1
- package/dist/crypto/tiers/enhanced.d.ts +0 -25
- package/dist/crypto/tiers/enhanced.d.ts.map +0 -1
- package/dist/crypto/tiers/enhanced.js +0 -56
- package/dist/crypto/tiers/enhanced.js.map +0 -1
- package/dist/crypto/tiers/maximum.d.ts +0 -19
- package/dist/crypto/tiers/maximum.d.ts.map +0 -1
- package/dist/crypto/tiers/maximum.js +0 -25
- package/dist/crypto/tiers/maximum.js.map +0 -1
- package/dist/crypto/tiers/standard.d.ts +0 -27
- package/dist/crypto/tiers/standard.d.ts.map +0 -1
- package/dist/crypto/tiers/standard.js +0 -155
- package/dist/crypto/tiers/standard.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../crypto/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../crypto/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,oBAAoB,IAAI,YAAY,EAAE,MAAM,qCAAqC,CAAC;AAC3F,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AACxC,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AACtG,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC"}
|
package/dist/crypto/index.js
CHANGED
|
@@ -1,15 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* chaoskb crypto module — knowledge-base blob primitives.
|
|
3
|
+
*
|
|
4
|
+
* Key-lifecycle (master-key wrapping, tiers, project keys, invites,
|
|
5
|
+
* TOFU pinning) lives in `@de-otio/keyring`. Pre-existing re-exports
|
|
6
|
+
* of those modules have been removed.
|
|
7
|
+
*/
|
|
1
8
|
export * from './types.js';
|
|
2
|
-
export { SecureBuffer } from '
|
|
9
|
+
export { SecureBuffer } from '@de-otio/crypto-envelope';
|
|
3
10
|
export { canonicalJson } from './canonical-json.js';
|
|
4
11
|
export { generateBlobId } from './blob-id.js';
|
|
5
12
|
export { aeadEncrypt, aeadDecrypt } from './aead.js';
|
|
6
13
|
export { deriveKey, deriveKeySet } from './hkdf.js';
|
|
7
|
-
export { deriveFromPassphrase as argon2Derive } from '
|
|
14
|
+
export { deriveFromPassphrase as argon2Derive } from '@de-otio/crypto-envelope/primitives';
|
|
8
15
|
export { constructAAD } from './aad.js';
|
|
9
16
|
export { computeCommitment, verifyCommitment } from './commitment.js';
|
|
10
17
|
export { encryptPayload, decryptEnvelope } from './envelope.js';
|
|
11
18
|
export { parseSSHPublicKey, ed25519ToX25519PublicKey, ed25519ToX25519SecretKey } from './ssh-keys.js';
|
|
12
19
|
export { EncryptionService } from './encryption-service.js';
|
|
13
|
-
export { KeyringService } from './keyring.js';
|
|
14
|
-
export { createProjectKey, unwrapProjectKey } from './project-keys.js';
|
|
15
20
|
//# sourceMappingURL=index.js.map
|
package/dist/crypto/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../crypto/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../crypto/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,oBAAoB,IAAI,YAAY,EAAE,MAAM,qCAAqC,CAAC;AAC3F,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AACxC,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AACtG,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC"}
|
|
@@ -1,19 +1,26 @@
|
|
|
1
|
-
import type { SSHKeyInfo } from './types.js';
|
|
2
1
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* SSH key helpers — thin re-exports over `@de-otio/keyring`'s
|
|
3
|
+
* canonical implementations. Kept so existing chaoskb imports
|
|
4
|
+
* (`parseSSHPublicKey`, `ed25519ToX25519*`) continue to work.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
6
|
+
* New code should import directly from `@de-otio/keyring`.
|
|
7
7
|
*/
|
|
8
|
+
import { sshFingerprint, type SshKeyType } from '@de-otio/keyring';
|
|
9
|
+
import type { SSHKeyInfo } from './types.js';
|
|
10
|
+
/** Back-compat alias for `parseSshPublicKey`. */
|
|
8
11
|
export declare function parseSSHPublicKey(keyString: string): SSHKeyInfo;
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
*/
|
|
12
|
+
export { sshFingerprint };
|
|
13
|
+
export type { SshKeyType };
|
|
14
|
+
/** Convert an Ed25519 public key to X25519 (Curve25519). */
|
|
13
15
|
export declare function ed25519ToX25519PublicKey(ed25519PublicKey: Uint8Array): Uint8Array;
|
|
14
16
|
/**
|
|
15
|
-
* Convert an Ed25519 secret key to X25519
|
|
16
|
-
*
|
|
17
|
+
* Convert an Ed25519 secret key to X25519 secret key.
|
|
18
|
+
*
|
|
19
|
+
* Note: keyring's canonical implementation returns an `ISecureBuffer`
|
|
20
|
+
* (design-review B5 fix). For back-compat with the old chaoskb signature
|
|
21
|
+
* we copy the bytes into a plain `Uint8Array` and dispose the secure
|
|
22
|
+
* handle. Callers that want the secure-buffer form should import from
|
|
23
|
+
* `@de-otio/keyring` directly.
|
|
17
24
|
*/
|
|
18
25
|
export declare function ed25519ToX25519SecretKey(ed25519SecretKey: Uint8Array): Uint8Array;
|
|
19
26
|
//# sourceMappingURL=ssh-keys.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ssh-keys.d.ts","sourceRoot":"","sources":["../../crypto/ssh-keys.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ssh-keys.d.ts","sourceRoot":"","sources":["../../crypto/ssh-keys.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAEL,cAAc,EAGd,KAAK,UAAU,EAEhB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,UAAU,EAAc,MAAM,YAAY,CAAC;AAEzD,iDAAiD;AACjD,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,UAAU,CAQ/D;AAED,OAAO,EAAE,cAAc,EAAE,CAAC;AAC1B,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B,4DAA4D;AAC5D,wBAAgB,wBAAwB,CAAC,gBAAgB,EAAE,UAAU,GAAG,UAAU,CAEjF;AAED;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,gBAAgB,EAAE,UAAU,GAAG,UAAU,CAOjF"}
|
package/dist/crypto/ssh-keys.js
CHANGED
|
@@ -1,122 +1,42 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
|
-
import sodium from 'sodium-native';
|
|
3
1
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
2
|
+
* SSH key helpers — thin re-exports over `@de-otio/keyring`'s
|
|
3
|
+
* canonical implementations. Kept so existing chaoskb imports
|
|
4
|
+
* (`parseSSHPublicKey`, `ed25519ToX25519*`) continue to work.
|
|
6
5
|
*
|
|
7
|
-
*
|
|
6
|
+
* New code should import directly from `@de-otio/keyring`.
|
|
8
7
|
*/
|
|
8
|
+
import { parseSshPublicKey, sshFingerprint, ed25519ToX25519PublicKey as kED25519ToX25519PublicKey, ed25519ToX25519SecretKey as kED25519ToX25519SecretKey, } from '@de-otio/keyring';
|
|
9
|
+
/** Back-compat alias for `parseSshPublicKey`. */
|
|
9
10
|
export function parseSSHPublicKey(keyString) {
|
|
10
|
-
const
|
|
11
|
-
const parts = trimmed.split(/\s+/);
|
|
12
|
-
if (parts.length < 2) {
|
|
13
|
-
throw new Error('Invalid SSH public key format: expected "<type> <base64> [comment]"');
|
|
14
|
-
}
|
|
15
|
-
const typeStr = parts[0];
|
|
16
|
-
const base64Blob = parts[1];
|
|
17
|
-
const comment = parts.length > 2 ? parts.slice(2).join(' ') : undefined;
|
|
18
|
-
let type;
|
|
19
|
-
if (typeStr === 'ssh-ed25519') {
|
|
20
|
-
type = 'ed25519';
|
|
21
|
-
}
|
|
22
|
-
else if (typeStr === 'ssh-rsa') {
|
|
23
|
-
type = 'rsa';
|
|
24
|
-
}
|
|
25
|
-
else {
|
|
26
|
-
throw new Error(`Unsupported SSH key type: ${typeStr}`);
|
|
27
|
-
}
|
|
28
|
-
// Base64-decode the key blob
|
|
29
|
-
const blob = Buffer.from(base64Blob, 'base64');
|
|
30
|
-
// Parse the SSH wire format: length-prefixed strings
|
|
31
|
-
const publicKeyBytes = extractPublicKeyFromBlob(blob, type);
|
|
32
|
-
// Compute SHA-256 fingerprint
|
|
33
|
-
const hash = createHash('sha256').update(blob).digest();
|
|
34
|
-
const fingerprint = 'SHA256:' + hash.toString('base64').replace(/=+$/, '');
|
|
11
|
+
const parsed = parseSshPublicKey(keyString);
|
|
35
12
|
return {
|
|
36
|
-
type,
|
|
37
|
-
publicKeyBytes,
|
|
38
|
-
fingerprint,
|
|
39
|
-
...(comment !== undefined && { comment }),
|
|
13
|
+
type: parsed.type,
|
|
14
|
+
publicKeyBytes: parsed.publicKeyBytes,
|
|
15
|
+
fingerprint: parsed.fingerprint,
|
|
16
|
+
...(parsed.comment !== undefined && { comment: parsed.comment }),
|
|
40
17
|
};
|
|
41
18
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
*
|
|
45
|
-
* SSH wire format: repeated [4-byte big-endian length][data]
|
|
46
|
-
* - For ed25519: [type string][32-byte public key]
|
|
47
|
-
* - For RSA: [type string][exponent][modulus]
|
|
48
|
-
*/
|
|
49
|
-
function extractPublicKeyFromBlob(blob, type) {
|
|
50
|
-
let offset = 0;
|
|
51
|
-
function readString() {
|
|
52
|
-
if (offset + 4 > blob.length) {
|
|
53
|
-
throw new Error('SSH key blob: unexpected end of data reading length');
|
|
54
|
-
}
|
|
55
|
-
const len = blob.readUInt32BE(offset);
|
|
56
|
-
offset += 4;
|
|
57
|
-
if (offset + len > blob.length) {
|
|
58
|
-
throw new Error('SSH key blob: unexpected end of data reading string');
|
|
59
|
-
}
|
|
60
|
-
const data = blob.subarray(offset, offset + len);
|
|
61
|
-
offset += len;
|
|
62
|
-
return data;
|
|
63
|
-
}
|
|
64
|
-
// First field: key type string
|
|
65
|
-
const typeField = readString().toString('ascii');
|
|
66
|
-
if (type === 'ed25519') {
|
|
67
|
-
if (typeField !== 'ssh-ed25519') {
|
|
68
|
-
throw new Error(`SSH key type mismatch: expected ssh-ed25519, got ${typeField}`);
|
|
69
|
-
}
|
|
70
|
-
// Second field: 32-byte public key
|
|
71
|
-
const pubkey = readString();
|
|
72
|
-
if (pubkey.length !== 32) {
|
|
73
|
-
throw new Error(`Ed25519 public key must be 32 bytes, got ${pubkey.length}`);
|
|
74
|
-
}
|
|
75
|
-
return new Uint8Array(pubkey);
|
|
76
|
-
}
|
|
77
|
-
else if (type === 'rsa') {
|
|
78
|
-
if (typeField !== 'ssh-rsa') {
|
|
79
|
-
throw new Error(`SSH key type mismatch: expected ssh-rsa, got ${typeField}`);
|
|
80
|
-
}
|
|
81
|
-
// RSA blob: [exponent][modulus]
|
|
82
|
-
// For RSA we return the full blob (exponent + modulus) since RSA operations
|
|
83
|
-
// need the full public key structure. The raw blob minus type prefix.
|
|
84
|
-
const exponent = readString();
|
|
85
|
-
const modulus = readString();
|
|
86
|
-
// Build a buffer with the exponent and modulus in SSH wire format
|
|
87
|
-
// This is what Node.js crypto needs for RSA operations
|
|
88
|
-
const result = Buffer.alloc(4 + exponent.length + 4 + modulus.length);
|
|
89
|
-
let pos = 0;
|
|
90
|
-
result.writeUInt32BE(exponent.length, pos);
|
|
91
|
-
pos += 4;
|
|
92
|
-
exponent.copy(result, pos);
|
|
93
|
-
pos += exponent.length;
|
|
94
|
-
result.writeUInt32BE(modulus.length, pos);
|
|
95
|
-
pos += 4;
|
|
96
|
-
modulus.copy(result, pos);
|
|
97
|
-
return new Uint8Array(result);
|
|
98
|
-
}
|
|
99
|
-
throw new Error(`Unsupported key type: ${type}`);
|
|
100
|
-
}
|
|
101
|
-
/**
|
|
102
|
-
* Convert an Ed25519 public key to X25519 (Curve25519) public key.
|
|
103
|
-
* Uses sodium crypto_sign_ed25519_pk_to_curve25519.
|
|
104
|
-
*/
|
|
19
|
+
export { sshFingerprint };
|
|
20
|
+
/** Convert an Ed25519 public key to X25519 (Curve25519). */
|
|
105
21
|
export function ed25519ToX25519PublicKey(ed25519PublicKey) {
|
|
106
|
-
|
|
107
|
-
const ed25519Buffer = Buffer.from(ed25519PublicKey);
|
|
108
|
-
sodium.crypto_sign_ed25519_pk_to_curve25519(x25519PublicKey, ed25519Buffer);
|
|
109
|
-
return new Uint8Array(x25519PublicKey);
|
|
22
|
+
return kED25519ToX25519PublicKey(ed25519PublicKey);
|
|
110
23
|
}
|
|
111
24
|
/**
|
|
112
|
-
* Convert an Ed25519 secret key to X25519
|
|
113
|
-
*
|
|
25
|
+
* Convert an Ed25519 secret key to X25519 secret key.
|
|
26
|
+
*
|
|
27
|
+
* Note: keyring's canonical implementation returns an `ISecureBuffer`
|
|
28
|
+
* (design-review B5 fix). For back-compat with the old chaoskb signature
|
|
29
|
+
* we copy the bytes into a plain `Uint8Array` and dispose the secure
|
|
30
|
+
* handle. Callers that want the secure-buffer form should import from
|
|
31
|
+
* `@de-otio/keyring` directly.
|
|
114
32
|
*/
|
|
115
33
|
export function ed25519ToX25519SecretKey(ed25519SecretKey) {
|
|
116
|
-
const
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
34
|
+
const secure = kED25519ToX25519SecretKey(ed25519SecretKey);
|
|
35
|
+
try {
|
|
36
|
+
return new Uint8Array(Buffer.from(secure.buffer));
|
|
37
|
+
}
|
|
38
|
+
finally {
|
|
39
|
+
secure.dispose();
|
|
40
|
+
}
|
|
121
41
|
}
|
|
122
42
|
//# sourceMappingURL=ssh-keys.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ssh-keys.js","sourceRoot":"","sources":["../../crypto/ssh-keys.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"ssh-keys.js","sourceRoot":"","sources":["../../crypto/ssh-keys.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EACL,iBAAiB,EACjB,cAAc,EACd,wBAAwB,IAAI,yBAAyB,EACrD,wBAAwB,IAAI,yBAAyB,GAGtD,MAAM,kBAAkB,CAAC;AAG1B,iDAAiD;AACjD,MAAM,UAAU,iBAAiB,CAAC,SAAiB;IACjD,MAAM,MAAM,GAAiB,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAC1D,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAkB;QAC/B,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;KACjE,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,cAAc,EAAE,CAAC;AAG1B,4DAA4D;AAC5D,MAAM,UAAU,wBAAwB,CAAC,gBAA4B;IACnE,OAAO,yBAAyB,CAAC,gBAAgB,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,wBAAwB,CAAC,gBAA4B;IACnE,MAAM,MAAM,GAAG,yBAAyB,CAAC,gBAAgB,CAAC,CAAC;IAC3D,IAAI,CAAC;QACH,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACpD,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;AACH,CAAC"}
|
package/dist/crypto/types.d.ts
CHANGED
|
@@ -1,20 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
export
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
/** Whether the buffer has been zeroed and disposed */
|
|
11
|
-
readonly isDisposed: boolean;
|
|
12
|
-
/** Zero the buffer contents and release memory */
|
|
13
|
-
dispose(): void;
|
|
14
|
-
}
|
|
15
|
-
/** Supported encryption algorithms */
|
|
16
|
-
export type Algorithm = 'XChaCha20-Poly1305' | 'AES-256-GCM';
|
|
17
|
-
/** Key identifier for derived keys */
|
|
1
|
+
import type { Algorithm as _Algorithm, AnyEnvelope as _AnyEnvelope, EnvelopeV1 as _EnvelopeV1, EnvelopeV2 as _EnvelopeV2, ISecureBuffer as _ISecureBuffer } from '@de-otio/crypto-envelope';
|
|
2
|
+
export type ISecureBuffer = _ISecureBuffer;
|
|
3
|
+
export type Algorithm = _Algorithm;
|
|
4
|
+
export type EnvelopeV2 = _EnvelopeV2;
|
|
5
|
+
export type AnyEnvelope = _AnyEnvelope;
|
|
6
|
+
/** Envelope wire type (v1). Alias of the package's `EnvelopeV1` — same
|
|
7
|
+
* shape, kept under the historical chaoskb name. */
|
|
8
|
+
export type Envelope = _EnvelopeV1;
|
|
9
|
+
/** Key identifier for derived keys. chaoskb-specific vocabulary. */
|
|
18
10
|
export type KeyId = 'CEK' | 'MEK' | 'EEK';
|
|
19
11
|
/** Security tier for key management */
|
|
20
12
|
export declare enum SecurityTier {
|
|
@@ -38,58 +30,14 @@ export interface DerivedKeySet {
|
|
|
38
30
|
}
|
|
39
31
|
/** SSH key type */
|
|
40
32
|
export type SSHKeyType = 'ed25519' | 'rsa';
|
|
41
|
-
/** Parsed SSH key information
|
|
33
|
+
/** Parsed SSH key information — chaoskb-local shape (kept for back-compat;
|
|
34
|
+
* new code should use `SshPublicKey` from `@de-otio/keyring`). */
|
|
42
35
|
export interface SSHKeyInfo {
|
|
43
36
|
type: SSHKeyType;
|
|
44
37
|
publicKeyBytes: Uint8Array;
|
|
45
38
|
fingerprint: string;
|
|
46
39
|
comment?: string;
|
|
47
40
|
}
|
|
48
|
-
/** Encryption envelope v1 wire format */
|
|
49
|
-
export interface Envelope {
|
|
50
|
-
/** Envelope version (must be 1) */
|
|
51
|
-
v: 1;
|
|
52
|
-
/** Opaque blob identifier (b_ prefix + base62) */
|
|
53
|
-
id: string;
|
|
54
|
-
/** ISO 8601 timestamp (server-generated) */
|
|
55
|
-
ts: string;
|
|
56
|
-
/** Encryption envelope */
|
|
57
|
-
enc: {
|
|
58
|
-
/** Algorithm identifier */
|
|
59
|
-
alg: Algorithm;
|
|
60
|
-
/** Key identifier */
|
|
61
|
-
kid: KeyId;
|
|
62
|
-
/** Base64-encoded: nonce || ciphertext || auth_tag */
|
|
63
|
-
ct: string;
|
|
64
|
-
/** Byte length of decoded ct */
|
|
65
|
-
'ct.len': number;
|
|
66
|
-
/** Base64-encoded HMAC-SHA256 key commitment */
|
|
67
|
-
commit: string;
|
|
68
|
-
};
|
|
69
|
-
}
|
|
70
|
-
/**
|
|
71
|
-
* Envelope v2 CBOR wire format — stores ct and commit as raw binary
|
|
72
|
-
* instead of base64, saving ~33% size on ciphertext.
|
|
73
|
-
*/
|
|
74
|
-
export interface EnvelopeV2 {
|
|
75
|
-
/** Envelope version (must be 2) */
|
|
76
|
-
v: 2;
|
|
77
|
-
/** Opaque blob identifier */
|
|
78
|
-
id: string;
|
|
79
|
-
/** ISO 8601 timestamp */
|
|
80
|
-
ts: string;
|
|
81
|
-
/** Encryption envelope */
|
|
82
|
-
enc: {
|
|
83
|
-
alg: Algorithm;
|
|
84
|
-
kid: KeyId;
|
|
85
|
-
/** Raw binary: nonce || ciphertext || auth_tag */
|
|
86
|
-
ct: Uint8Array;
|
|
87
|
-
/** HMAC-SHA256 key commitment (raw binary) */
|
|
88
|
-
commit: Uint8Array;
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
|
-
/** Union type for any supported envelope version */
|
|
92
|
-
export type AnyEnvelope = Envelope | EnvelopeV2;
|
|
93
41
|
/** Plaintext payload types */
|
|
94
42
|
export type PayloadType = 'source' | 'chunk' | 'canary';
|
|
95
43
|
/** Source payload (decrypted) */
|
|
@@ -129,13 +77,13 @@ export interface DecryptResult {
|
|
|
129
77
|
payload: Payload;
|
|
130
78
|
envelope: Envelope;
|
|
131
79
|
}
|
|
132
|
-
/**
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
80
|
+
/**
|
|
81
|
+
* Encryption service interface — used by sync/canary and the MCP server.
|
|
82
|
+
*
|
|
83
|
+
* The default implementation {@link EncryptionService} delegates to
|
|
84
|
+
* `@de-otio/crypto-envelope`'s `encryptV1` / `decryptV1` primitives and
|
|
85
|
+
* chaoskb's HKDF key-set derivation. Tests provide their own mocks.
|
|
86
|
+
*/
|
|
139
87
|
export interface IEncryptionService {
|
|
140
88
|
/** Generate a new random master key */
|
|
141
89
|
generateMasterKey(): ISecureBuffer;
|
|
@@ -148,22 +96,4 @@ export interface IEncryptionService {
|
|
|
148
96
|
/** Generate a blob ID */
|
|
149
97
|
generateBlobId(): string;
|
|
150
98
|
}
|
|
151
|
-
/** Key management service for a specific security tier */
|
|
152
|
-
export interface IKeyManager {
|
|
153
|
-
tier: SecurityTier;
|
|
154
|
-
/** Wrap master key for storage */
|
|
155
|
-
wrapMasterKey(masterKey: ISecureBuffer): Promise<Uint8Array>;
|
|
156
|
-
/** Unwrap master key from storage */
|
|
157
|
-
unwrapMasterKey(wrappedKey: Uint8Array): Promise<ISecureBuffer>;
|
|
158
|
-
}
|
|
159
|
-
/** Project key management */
|
|
160
|
-
export interface IProjectKeyManager {
|
|
161
|
-
/** Generate a new project key and wrap with personal master key */
|
|
162
|
-
createProjectKey(masterKey: ISecureBuffer): Promise<{
|
|
163
|
-
projectKey: ISecureBuffer;
|
|
164
|
-
wrappedKey: Uint8Array;
|
|
165
|
-
}>;
|
|
166
|
-
/** Unwrap a project key using personal master key */
|
|
167
|
-
unwrapProjectKey(wrappedKey: Uint8Array, masterKey: ISecureBuffer): Promise<ISecureBuffer>;
|
|
168
|
-
}
|
|
169
99
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../crypto/types.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../crypto/types.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,SAAS,IAAI,UAAU,EACvB,WAAW,IAAI,YAAY,EAC3B,UAAU,IAAI,WAAW,EACzB,UAAU,IAAI,WAAW,EACzB,aAAa,IAAI,cAAc,EAChC,MAAM,0BAA0B,CAAC;AAElC,MAAM,MAAM,aAAa,GAAG,cAAc,CAAC;AAC3C,MAAM,MAAM,SAAS,GAAG,UAAU,CAAC;AACnC,MAAM,MAAM,UAAU,GAAG,WAAW,CAAC;AACrC,MAAM,MAAM,WAAW,GAAG,YAAY,CAAC;AAEvC;qDACqD;AACrD,MAAM,MAAM,QAAQ,GAAG,WAAW,CAAC;AAInC,oEAAoE;AACpE,MAAM,MAAM,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;AAE1C,uCAAuC;AACvC,oBAAY,YAAY;IACtB,+EAA+E;IAC/E,QAAQ,aAAa;IACrB,yFAAyF;IACzF,QAAQ,aAAa;IACrB,kDAAkD;IAClD,OAAO,YAAY;CACpB;AAED,8CAA8C;AAC9C,MAAM,WAAW,aAAa;IAC5B,4DAA4D;IAC5D,UAAU,EAAE,aAAa,CAAC;IAC1B,8DAA8D;IAC9D,WAAW,EAAE,aAAa,CAAC;IAC3B,0EAA0E;IAC1E,YAAY,EAAE,aAAa,CAAC;IAC5B,4DAA4D;IAC5D,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED,mBAAmB;AACnB,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,KAAK,CAAC;AAE3C;mEACmE;AACnE,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,UAAU,CAAC;IACjB,cAAc,EAAE,UAAU,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,8BAA8B;AAC9B,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;AAExD,iCAAiC;AACjC,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,QAAQ,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,gCAAgC;AAChC,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,0CAA0C;AAC1C,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,mBAAmB,CAAC;CAC5B;AAED,4BAA4B;AAC5B,MAAM,MAAM,OAAO,GAAG,aAAa,GAAG,YAAY,GAAG,aAAa,CAAC;AAEnE,2BAA2B;AAC3B,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,QAAQ,CAAC;IACnB,gCAAgC;IAChC,KAAK,EAAE,UAAU,CAAC;CACnB;AAED,2BAA2B;AAC3B,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC,uCAAuC;IACvC,iBAAiB,IAAI,aAAa,CAAC;IACnC,yCAAyC;IACzC,UAAU,CAAC,SAAS,EAAE,aAAa,EAAE,IAAI,CAAC,EAAE,UAAU,GAAG,aAAa,CAAC;IACvE,yCAAyC;IACzC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,CAAC,EAAE,KAAK,GAAG,aAAa,CAAC;IAC3E,yCAAyC;IACzC,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,aAAa,GAAG,aAAa,CAAC;IAChE,yBAAyB;IACzB,cAAc,IAAI,MAAM,CAAC;CAC1B"}
|
package/dist/crypto/types.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
// Types shared with `@de-otio/crypto-envelope` are re-exported below; types
|
|
2
|
+
// unique to chaoskb (payload variants, SSH info, encryption-service
|
|
3
|
+
// interface) are declared here.
|
|
1
4
|
/** Security tier for key management */
|
|
2
5
|
export var SecurityTier;
|
|
3
6
|
(function (SecurityTier) {
|
package/dist/crypto/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../crypto/types.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../crypto/types.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,oEAAoE;AACpE,gCAAgC;AAwBhC,uCAAuC;AACvC,MAAM,CAAN,IAAY,YAOX;AAPD,WAAY,YAAY;IACtB,+EAA+E;IAC/E,qCAAqB,CAAA;IACrB,yFAAyF;IACzF,qCAAqB,CAAA;IACrB,kDAAkD;IAClD,mCAAmB,CAAA;AACrB,CAAC,EAPW,YAAY,KAAZ,YAAY,QAOvB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"content-pipeline.d.ts","sourceRoot":"","sources":["../../pipeline/content-pipeline.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"content-pipeline.d.ts","sourceRoot":"","sources":["../../pipeline/content-pipeline.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAQ9C,OAAO,KAAK,EACV,KAAK,EACL,aAAa,EACb,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACf,MAAM,YAAY,CAAC;AAEpB;;;;;GAKG;AACH,qBAAa,eAAgB,YAAW,gBAAgB;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;IACjD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;gBAExB,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,EAAE,QAAQ,EAAE,QAAQ;IAK/D,wDAAwD;IAClD,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwC7D,yCAAyC;IACnC,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkBlE,0CAA0C;IAC1C,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE;IAO5B,kCAAkC;IAC5B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAInD,kEAAkE;IAC5D,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAS5D,6EAA6E;IAC7E,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;CAItF"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { chunkText } from './chunker.js';
|
|
2
|
-
import { extractContent } from './extract.js';
|
|
2
|
+
import { extractContent, JsRenderRequiredError } from './extract.js';
|
|
3
3
|
import { extractFromFile as fileExtract } from './file-extract.js';
|
|
4
4
|
import { fetchUrl } from './fetch.js';
|
|
5
|
+
import { fetchUrlWithBrowser } from './fetch-browser.js';
|
|
5
6
|
import { safetyChecker } from './safety.js';
|
|
6
7
|
import { searchEmbeddings } from './search.js';
|
|
7
8
|
import { validateContent, validateFileContent } from './validate.js';
|
|
@@ -27,8 +28,23 @@ export class ContentPipeline {
|
|
|
27
28
|
}
|
|
28
29
|
}
|
|
29
30
|
const result = await fetchUrl(url, this.config);
|
|
30
|
-
|
|
31
|
-
|
|
31
|
+
let html = result.html;
|
|
32
|
+
let extracted;
|
|
33
|
+
try {
|
|
34
|
+
extracted = extractContent(html, result.finalUrl);
|
|
35
|
+
}
|
|
36
|
+
catch (err) {
|
|
37
|
+
if (err instanceof JsRenderRequiredError) {
|
|
38
|
+
// JavaScript-rendered page: re-fetch through headless Chromium and
|
|
39
|
+
// run the same extraction pipeline on the post-render HTML.
|
|
40
|
+
html = await fetchUrlWithBrowser(result.finalUrl);
|
|
41
|
+
extracted = extractContent(html, result.finalUrl);
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
throw err;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const issues = validateContent(html, extracted);
|
|
32
48
|
const errors = issues.filter((i) => i.severity === 'error');
|
|
33
49
|
if (errors.length > 0) {
|
|
34
50
|
throw new Error(errors.map((e) => e.message).join('; '));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"content-pipeline.js","sourceRoot":"","sources":["../../pipeline/content-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"content-pipeline.js","sourceRoot":"","sources":["../../pipeline/content-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,EAAE,eAAe,IAAI,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACnE,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAUrE;;;;;GAKG;AACH,MAAM,OAAO,eAAe;IACT,MAAM,CAA0B;IAChC,QAAQ,CAAW;IAEpC,YAAY,MAA+B,EAAE,QAAkB;QAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED,wDAAwD;IACxD,KAAK,CAAC,eAAe,CAAC,GAAW;QAC/B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAClC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACnD,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,gBAAgB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhD,IAAI,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACvB,IAAI,SAA2B,CAAC;QAChC,IAAI,CAAC;YACH,SAAS,GAAG,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,qBAAqB,EAAE,CAAC;gBACzC,mEAAmE;gBACnE,4DAA4D;gBAC5D,IAAI,GAAG,MAAM,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAClD,SAAS,GAAG,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAEhD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC;QAC5D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC;QAC7D,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,SAAS,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,yCAAyC;IACzC,KAAK,CAAC,eAAe,CAAC,QAAgB;QACpC,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC;QAC5D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC;QAC7D,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,SAAS,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,0CAA0C;IAC1C,KAAK,CAAC,IAAY;QAChB,OAAO,SAAS,CAAC,IAAI,EAAE;YACrB,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,IAAI,GAAG;YAC5C,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,EAAE;SAC/C,CAAC,CAAC;IACL,CAAC;IAED,kCAAkC;IAClC,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,kEAAkE;IAClE,KAAK,CAAC,WAAW,CAAC,MAAe;QAC/B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7E,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,GAAG,KAAK;YACR,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;YACrB,KAAK,EAAE,8BAA8B;SACtC,CAAC,CAAC,CAAC;IACN,CAAC;IAED,6EAA6E;IAC7E,MAAM,CAAC,KAAsB,EAAE,UAA6B,EAAE,IAAY;QACxE,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;QAC1D,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;CACF"}
|
|
@@ -5,6 +5,14 @@
|
|
|
5
5
|
* the main article content, stripped of navigation, ads, and boilerplate.
|
|
6
6
|
*/
|
|
7
7
|
import type { ExtractedContent } from './types.js';
|
|
8
|
+
/**
|
|
9
|
+
* Thrown when a page appears to require client-side JavaScript to render
|
|
10
|
+
* its content. Callers can catch this to invoke a headless-browser fallback.
|
|
11
|
+
*/
|
|
12
|
+
export declare class JsRenderRequiredError extends Error {
|
|
13
|
+
readonly url: string;
|
|
14
|
+
constructor(url: string);
|
|
15
|
+
}
|
|
8
16
|
/**
|
|
9
17
|
* Extract the main article content from an HTML string.
|
|
10
18
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../pipeline/extract.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../pipeline/extract.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD;;;GAGG;AACH,qBAAa,qBAAsB,SAAQ,KAAK;aAClB,GAAG,EAAE,MAAM;gBAAX,GAAG,EAAE,MAAM;CAOxC;AAoDD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,gBAAgB,CAqE1E"}
|
package/dist/pipeline/extract.js
CHANGED
|
@@ -7,6 +7,19 @@
|
|
|
7
7
|
import { basename } from 'node:path';
|
|
8
8
|
import { Readability } from '@mozilla/readability';
|
|
9
9
|
import { parseHTML } from 'linkedom';
|
|
10
|
+
/**
|
|
11
|
+
* Thrown when a page appears to require client-side JavaScript to render
|
|
12
|
+
* its content. Callers can catch this to invoke a headless-browser fallback.
|
|
13
|
+
*/
|
|
14
|
+
export class JsRenderRequiredError extends Error {
|
|
15
|
+
url;
|
|
16
|
+
constructor(url) {
|
|
17
|
+
super(`This page appears to require JavaScript to render its content (${url}). ` +
|
|
18
|
+
`Only the noscript fallback was captured.`);
|
|
19
|
+
this.url = url;
|
|
20
|
+
this.name = 'JsRenderRequiredError';
|
|
21
|
+
}
|
|
22
|
+
}
|
|
10
23
|
/**
|
|
11
24
|
* Regex matching inline `style` attribute values that visually hide an element.
|
|
12
25
|
*
|
|
@@ -67,8 +80,7 @@ export function extractContent(html, url) {
|
|
|
67
80
|
// SPA pages have <noscript> as their only meaningful content; once we strip it
|
|
68
81
|
// below, there's nothing left to extract and we'd get a confusing error.
|
|
69
82
|
if (looksLikeSpaHtml(html)) {
|
|
70
|
-
throw new
|
|
71
|
-
`Only the noscript fallback was captured. ChaosKB does not yet support JavaScript-rendered pages.`);
|
|
83
|
+
throw new JsRenderRequiredError(safeSourceLabel(url));
|
|
72
84
|
}
|
|
73
85
|
const { document } = parseHTML(html);
|
|
74
86
|
// Strip visually-hidden elements before Readability sees them.
|
|
@@ -112,8 +124,7 @@ export function extractContent(html, url) {
|
|
|
112
124
|
}
|
|
113
125
|
// Detect JavaScript-only SPA pages that didn't render
|
|
114
126
|
if (looksLikeJsOnlyPage(html, content)) {
|
|
115
|
-
throw new
|
|
116
|
-
`Only the noscript fallback was captured. ChaosKB does not yet support JavaScript-rendered pages.`);
|
|
127
|
+
throw new JsRenderRequiredError(safeSourceLabel(url));
|
|
117
128
|
}
|
|
118
129
|
const byteLength = Buffer.byteLength(content, 'utf-8');
|
|
119
130
|
return { title, content, url, byteLength };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../pipeline/extract.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAGrC;;;;;;GAMG;AACH,MAAM,eAAe,GACnB,oNAAoN,CAAC;AAEvN;;;;;;;;GAQG;AACH,SAAS,mBAAmB,CAAC,QAAa;IACxC,0DAA0D;IAC1D,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,EAAE,CAAC;QACvD,EAAE,CAAC,MAAM,EAAE,CAAC;IACd,CAAC;IAED,8CAA8C;IAC9C,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,EAAE,CAAC;QACvD,EAAE,CAAC,MAAM,EAAE,CAAC;IACd,CAAC;IAED,0CAA0C;IAC1C,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,gBAAgB,CAAC,sBAAsB,CAAC,EAAE,CAAC;QACnE,EAAE,CAAC,MAAM,EAAE,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC7C,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,EAAE,CAAC,MAAM,EAAE,CAAC;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;IACxE,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,GAAW;IACtD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,sEAAsE;IACtE,+EAA+E;IAC/E,yEAAyE;IACzE,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,
|
|
1
|
+
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../pipeline/extract.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAGrC;;;GAGG;AACH,MAAM,OAAO,qBAAsB,SAAQ,KAAK;IAClB;IAA5B,YAA4B,GAAW;QACrC,KAAK,CACH,kEAAkE,GAAG,KAAK;YAC1E,0CAA0C,CAC3C,CAAC;QAJwB,QAAG,GAAH,GAAG,CAAQ;QAKrC,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;IACtC,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,eAAe,GACnB,oNAAoN,CAAC;AAEvN;;;;;;;;GAQG;AACH,SAAS,mBAAmB,CAAC,QAAa;IACxC,0DAA0D;IAC1D,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,EAAE,CAAC;QACvD,EAAE,CAAC,MAAM,EAAE,CAAC;IACd,CAAC;IAED,8CAA8C;IAC9C,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,EAAE,CAAC;QACvD,EAAE,CAAC,MAAM,EAAE,CAAC;IACd,CAAC;IAED,0CAA0C;IAC1C,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,gBAAgB,CAAC,sBAAsB,CAAC,EAAE,CAAC;QACnE,EAAE,CAAC,MAAM,EAAE,CAAC;IACd,CAAC;IAED,6DAA6D;IAC7D,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC7C,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,EAAE,CAAC,MAAM,EAAE,CAAC;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;IACxE,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,GAAW;IACtD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,sEAAsE;IACtE,+EAA+E;IAC/E,yEAAyE;IACzE,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,qBAAqB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAErC,+DAA+D;IAC/D,+EAA+E;IAC/E,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAE9B,iCAAiC;IACjC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAe,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;IAE/B,IAAI,KAAa,CAAC;IAClB,IAAI,UAAkB,CAAC;IAEvB,IAAI,OAAO,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5E,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAC5B,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,qEAAqE;QACrE,yEAAyE;QACzE,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,IAAI,gBAAgB,CAAC;QACxF,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;QACzD,mBAAmB,CAAC,WAAW,CAAC,CAAC;QACjC,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;YAC/D,EAAE,CAAC,MAAM,EAAE,CAAC;QACd,CAAC;QACD,MAAM,IAAI,GAAG,WAAW,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAC/C,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAEhD,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,+BAA+B,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,KAAK,GAAG,EAAE,CAAC;IACb,CAAC;IAED,qDAAqD;IACrD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAChD,KAAK,GAAG,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC7C,CAAC;IAED,0DAA0D;IAC1D,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;IAEtC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,sDAAsD;IACtD,IAAI,mBAAmB,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,qBAAqB,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAEvD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC;AAC7C,CAAC;AAED,0DAA0D;AAC1D,MAAM,iBAAiB,GAAG;IACxB,gCAAgC;IAChC,2BAA2B;IAC3B,yBAAyB;IACzB,yBAAyB;IACzB,+BAA+B;IAC/B,oCAAoC;IACpC,uCAAuC;IACvC,6BAA6B;IAC7B,qCAAqC;IACrC,gCAAgC;CACjC,CAAC;AAEF;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC;;;;;;GAMG;AACH,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,WAAW,GAAG,qCAAqC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,IAAI,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC;IAE/B,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACvE,IAAI,CAAC,kBAAkB;QAAE,OAAO,KAAK,CAAC;IAEtC,MAAM,UAAU,GACd,sEAAsE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEpF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,mBAAmB,CAAC,IAAY,EAAE,aAAqB;IAC9D,wFAAwF;IACxF,MAAM,kBAAkB,GACtB,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACpD,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9C,IAAI,CAAC,kBAAkB;QAAE,OAAO,KAAK,CAAC;IAEtC,mEAAmE;IACnE,IAAI,aAAa,CAAC,MAAM,GAAG,qBAAqB;QAAE,OAAO,IAAI,CAAC;IAE9D,8DAA8D;IAC9D,8DAA8D;IAC9D,MAAM,UAAU,GACd,sEAAsE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEpF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI;SACR,OAAO,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAgC,8CAA8C;SAC9G,OAAO,CAAC,kDAAkD,EAAE,EAAE,CAAC,CAAC,4CAA4C;SAC5G,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAO,iCAAiC;SAC/D,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAI,8BAA8B;SAC5D,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAM,iBAAiB;SAC/C,IAAI,EAAE,CAAC;AACZ,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Headless-browser fallback for JavaScript-rendered pages.
|
|
3
|
+
*
|
|
4
|
+
* When `extractContent` throws `JsRenderRequiredError`, the content pipeline
|
|
5
|
+
* calls `fetchUrlWithBrowser` to re-fetch the URL through a Chromium instance
|
|
6
|
+
* that executes the page's JavaScript, then feeds the rendered HTML back into
|
|
7
|
+
* the normal extraction path.
|
|
8
|
+
*
|
|
9
|
+
* A module-level singleton Browser is launched lazily on first use and
|
|
10
|
+
* self-closes after `IDLE_SHUTDOWN_MS` of inactivity, amortizing the 2–5 s
|
|
11
|
+
* cold start across sequential ingestions while releasing the Chromium
|
|
12
|
+
* subprocess (~200 MB resident) when the fallback isn't active.
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Fetch a URL through a headless Chromium instance, executing JavaScript and
|
|
16
|
+
* returning the post-render HTML.
|
|
17
|
+
*
|
|
18
|
+
* @param url - The URL to render. SSRF-validated before the browser launch.
|
|
19
|
+
* @returns The fully rendered HTML as a string.
|
|
20
|
+
* @throws If SSRF validation fails, navigation times out, or the rendered
|
|
21
|
+
* HTML exceeds `MAX_RESPONSE_BYTES`.
|
|
22
|
+
*/
|
|
23
|
+
export declare function fetchUrlWithBrowser(url: string): Promise<string>;
|
|
24
|
+
/**
|
|
25
|
+
* Testing hook: tear down the singleton browser and cancel any pending idle
|
|
26
|
+
* shutdown. Not called from production code.
|
|
27
|
+
*/
|
|
28
|
+
export declare function _resetBrowserSingletonForTests(): Promise<void>;
|
|
29
|
+
//# sourceMappingURL=fetch-browser.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch-browser.d.ts","sourceRoot":"","sources":["../../pipeline/fetch-browser.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAkCH;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA2BtE;AAED;;;GAGG;AACH,wBAAsB,8BAA8B,IAAI,OAAO,CAAC,IAAI,CAAC,CAapE"}
|