@hraness/sponge 0.6.8 → 0.6.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/dist/capture/worker.js +154 -152
- package/dist/contracts/index.d.ts +2 -0
- package/dist/src/acquisition/capture.d.ts +4 -0
- package/dist/src/acquisition/capture.js +8 -2
- package/dist/src/acquisition/child-request.d.ts +5 -0
- package/dist/src/acquisition/normalize.js +3 -0
- package/dist/src/app/article.d.ts +2 -0
- package/dist/src/app/article.js +17 -3
- package/dist/src/browser/bridge.d.ts +48 -0
- package/dist/src/browser/bridge.js +90 -0
- package/dist/src/browser/host.d.ts +10 -0
- package/dist/src/browser/host.js +147 -0
- package/dist/src/browser/setup.d.ts +26 -0
- package/dist/src/browser/setup.js +103 -0
- package/dist/src/capture/types.d.ts +1 -1
- package/dist/src/cli/commands.js +84 -12
- package/dist/src/errors.d.ts +1 -0
- package/dist/src/errors.js +2 -0
- package/dist/src/pdf/handler.js +1 -3
- package/dist/src/store/queue.d.ts +2 -2
- package/dist/src/store/queue.js +7 -4
- package/dist/src/store/types.d.ts +5 -2
- package/dist/src/worker/basic.js +15 -0
- package/extension/background.js +180 -0
- package/extension/manifest.json +25 -0
- package/extension/popup.html +85 -0
- package/extension/popup.js +39 -0
- package/native/runtime-manifest.json +5 -5
- package/package.json +2 -1
|
@@ -60,6 +60,8 @@ export interface CaptureResult extends Envelope {
|
|
|
60
60
|
text: 'complete' | 'partial';
|
|
61
61
|
images: 'complete' | 'partial' | 'not-requested';
|
|
62
62
|
};
|
|
63
|
+
/** The page came through the reader's own browser and sign-ins. */
|
|
64
|
+
browser?: true;
|
|
63
65
|
}
|
|
64
66
|
export interface Transcript extends Envelope {
|
|
65
67
|
source: 'human-captions' | 'automatic-captions' | 'local-asr';
|
|
@@ -11,6 +11,10 @@ export interface CaptureRequest {
|
|
|
11
11
|
sourceUrl: string;
|
|
12
12
|
requestId: string;
|
|
13
13
|
connection?: CaptureConnection;
|
|
14
|
+
browser?: {
|
|
15
|
+
socket: string;
|
|
16
|
+
mode: 'always' | 'fallback';
|
|
17
|
+
};
|
|
14
18
|
signal?: AbortSignal;
|
|
15
19
|
deadlineMs?: number;
|
|
16
20
|
allowPrivateNetwork?: boolean;
|
|
@@ -12,10 +12,15 @@ function validSource(sourceUrl) {
|
|
|
12
12
|
}
|
|
13
13
|
}
|
|
14
14
|
export function captureArticle(request) {
|
|
15
|
-
const endpoint = request.connection?.endpoint;
|
|
15
|
+
const endpoint = request.connection?.endpoint, browser = request.browser;
|
|
16
16
|
return superviseChild({
|
|
17
17
|
...request,
|
|
18
|
-
valid: validSource(request.sourceUrl) &&
|
|
18
|
+
valid: validSource(request.sourceUrl) &&
|
|
19
|
+
(endpoint === undefined || loopbackEndpoint.test(endpoint)) &&
|
|
20
|
+
(browser === undefined ||
|
|
21
|
+
(browser.socket.startsWith('/') &&
|
|
22
|
+
browser.socket.length < 104 &&
|
|
23
|
+
['always', 'fallback'].includes(browser.mode))),
|
|
19
24
|
prepare: async ({ output, deadline }) => ({
|
|
20
25
|
schemaVersion: 1,
|
|
21
26
|
kind: 'article',
|
|
@@ -28,6 +33,7 @@ export function captureArticle(request) {
|
|
|
28
33
|
maxTotalAssetBytes: 500 * 1024 ** 2,
|
|
29
34
|
allowPrivateNetwork: request.allowPrivateNetwork === true,
|
|
30
35
|
...(endpoint === undefined ? {} : { cdp: endpoint }),
|
|
36
|
+
...(browser === undefined ? {} : { browser: { socket: browser.socket, mode: browser.mode } }),
|
|
31
37
|
}),
|
|
32
38
|
verify: (bundleRoot) => normalizeBundle({ bundleRoot, requestId: request.requestId, sourceUrl: request.sourceUrl }),
|
|
33
39
|
});
|
|
@@ -14,6 +14,11 @@ export interface ArticleChildRequest extends ChildRequestBase {
|
|
|
14
14
|
maxHtmlBytes: number;
|
|
15
15
|
/** A loopback DevTools endpoint of the user's attached signed-in browser. */
|
|
16
16
|
cdp?: string;
|
|
17
|
+
/** The Sponge browser bridge socket: `always` captures only through it, `fallback` when the public page is thin. */
|
|
18
|
+
browser?: {
|
|
19
|
+
socket: string;
|
|
20
|
+
mode: 'always' | 'fallback';
|
|
21
|
+
};
|
|
17
22
|
}
|
|
18
23
|
export interface PdfChildRequest extends ChildRequestBase {
|
|
19
24
|
kind: 'pdf';
|
|
@@ -93,6 +93,9 @@ export async function normalizeBundle(input) {
|
|
|
93
93
|
extractor: { ...captureExtractor },
|
|
94
94
|
manifestSha256: hash(manifestBytes),
|
|
95
95
|
completeness: { text, images: imageStatus },
|
|
96
|
+
...(/^browser-(?:fetch|render)$/.test(String(manifest.acquisition?.method))
|
|
97
|
+
? { browser: true }
|
|
98
|
+
: {}),
|
|
96
99
|
};
|
|
97
100
|
validateCapture(capture);
|
|
98
101
|
return capture;
|
|
@@ -16,6 +16,8 @@ export interface ArticleOptions {
|
|
|
16
16
|
checkpoint?: (stage: JobCheckpoint['stage']) => Promise<void>;
|
|
17
17
|
runProcess?: CaptureRequest['runProcess'];
|
|
18
18
|
connection?: CaptureConnection;
|
|
19
|
+
/** The Sponge browser bridge socket: a thin public page is retried there, and `--browser` sources only use it. */
|
|
20
|
+
browser?: string;
|
|
19
21
|
signal?: AbortSignal;
|
|
20
22
|
allowPrivateNetwork?: boolean;
|
|
21
23
|
capture?: (request: CaptureRequest) => Promise<CaptureOutcome>;
|
package/dist/src/app/article.js
CHANGED
|
@@ -25,9 +25,7 @@ export function ingestArticle(store, token, options) {
|
|
|
25
25
|
if (!source || source.kind !== 'article')
|
|
26
26
|
throw new SpongeError('invalid-input');
|
|
27
27
|
// An archived copy is public: it is fetched directly, never through a signed-in browser.
|
|
28
|
-
const snapshot = source.archive
|
|
29
|
-
? await (options.snapshot ?? waybackSnapshot)(source.locator, token.attemptId)
|
|
30
|
-
: null;
|
|
28
|
+
const snapshot = source.via === 'archive' ? await (options.snapshot ?? waybackSnapshot)(source.locator, token.attemptId) : null;
|
|
31
29
|
if (snapshot && 'ok' in snapshot)
|
|
32
30
|
return snapshot;
|
|
33
31
|
const fetched = snapshot?.url ?? source.locator;
|
|
@@ -37,6 +35,10 @@ export function ingestArticle(store, token, options) {
|
|
|
37
35
|
sourceUrl: fetched,
|
|
38
36
|
requestId: token.attemptId,
|
|
39
37
|
connection: snapshot ? undefined : options.connection,
|
|
38
|
+
// An archived copy or an attached connection already names its route; the browser is never asked for it.
|
|
39
|
+
...(snapshot || options.connection || !options.browser
|
|
40
|
+
? {}
|
|
41
|
+
: { browser: { socket: options.browser, mode: source.via === 'browser' ? 'always' : 'fallback' } }),
|
|
40
42
|
signal: options.signal,
|
|
41
43
|
allowPrivateNetwork: options.allowPrivateNetwork,
|
|
42
44
|
});
|
|
@@ -83,6 +85,8 @@ export function ingestArticle(store, token, options) {
|
|
|
83
85
|
assets: pdf.assets,
|
|
84
86
|
warnings: pdf.warnings,
|
|
85
87
|
});
|
|
88
|
+
if (capture.browser)
|
|
89
|
+
keepPrivate(store, source.itemId);
|
|
86
90
|
return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status: pdf.status };
|
|
87
91
|
}
|
|
88
92
|
// Without the paper, the landing page is still what the reader saved.
|
|
@@ -118,9 +122,19 @@ export function ingestArticle(store, token, options) {
|
|
|
118
122
|
assets,
|
|
119
123
|
warnings: [...new Set([...capture.warnings, ...(localized.missingImages ? ['partial'] : [])])],
|
|
120
124
|
});
|
|
125
|
+
if (capture.browser)
|
|
126
|
+
keepPrivate(store, source.itemId);
|
|
121
127
|
return { ok: true, itemId: source.itemId, revisionId: revision.revisionId, status };
|
|
122
128
|
});
|
|
123
129
|
}
|
|
130
|
+
/**
|
|
131
|
+
* A page read with the reader's sign-ins is private, even when the item was shared before: what the browser read may
|
|
132
|
+
* be the subscriber's text. `sponge visibility ITEM public` shares it again deliberately.
|
|
133
|
+
*/
|
|
134
|
+
function keepPrivate(store, itemId) {
|
|
135
|
+
if (store.identity(itemId).visibility.override !== 'private')
|
|
136
|
+
store.setVisibility(itemId, 'private');
|
|
137
|
+
}
|
|
124
138
|
/** Prose words under which a page that names its paper's PDF is only a landing page: a title, authors and abstract. */
|
|
125
139
|
const landingPageWords = 1000;
|
|
126
140
|
/**
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Sponge browser bridge. The Sponge extension in the reader's own browser opens a native-messaging port; the
|
|
3
|
+
* browser starts `host.ts`, which listens on a private local socket. A capture asks over that socket, and the
|
|
4
|
+
* extension loads the page with the reader's own sign-ins. Cookies and passwords never leave the browser: Sponge
|
|
5
|
+
* receives only the page.
|
|
6
|
+
*/
|
|
7
|
+
export declare const hostName = "com.hraness.sponge";
|
|
8
|
+
/** Derived from the public key in extension/manifest.json, so an unpacked or store install keeps one identity. */
|
|
9
|
+
export declare const extensionId = "pcekakbinokokbcfdpgdljpeoiceoejf";
|
|
10
|
+
/** `fetch` asks with the reader's cookies and reads the HTML; `render` loads the page in a background window. */
|
|
11
|
+
export type BrowserOperation = 'fetch' | 'render';
|
|
12
|
+
export interface BrowserPage {
|
|
13
|
+
ok: true;
|
|
14
|
+
html: string;
|
|
15
|
+
finalUrl: string;
|
|
16
|
+
title?: string;
|
|
17
|
+
}
|
|
18
|
+
/** `disabled`: the reader turned background captures off. `offline`: no browser has the extension running. */
|
|
19
|
+
export interface BrowserRefusal {
|
|
20
|
+
ok: false;
|
|
21
|
+
code: 'disabled' | 'offline' | 'failed' | 'timeout' | 'not-html' | 'too-large';
|
|
22
|
+
status?: number;
|
|
23
|
+
}
|
|
24
|
+
export interface BrowserStatus {
|
|
25
|
+
ok: true;
|
|
26
|
+
extension: string;
|
|
27
|
+
background: boolean;
|
|
28
|
+
}
|
|
29
|
+
/** One bridge per library: a short path, because a Unix socket path is limited to 104 bytes on macOS. */
|
|
30
|
+
export declare function bridgeSocket(libraryRoot: string, home?: string): string;
|
|
31
|
+
/** Native messaging frames: a 32-bit length in the host's byte order (little-endian on every supported Mac), then JSON. */
|
|
32
|
+
export declare function encodeFrame(message: unknown): Buffer;
|
|
33
|
+
/** Splits complete frames off `buffer`; the rest waits for more bytes. A frame over `maxBytes` is refused. */
|
|
34
|
+
export declare function decodeFrames(buffer: Buffer, maxBytes: number): {
|
|
35
|
+
messages: unknown[];
|
|
36
|
+
rest: Buffer;
|
|
37
|
+
};
|
|
38
|
+
/** Requests over the socket are one JSON line each way; the bridge closes after its answer. */
|
|
39
|
+
export declare function askBridge(socket: string, request: {
|
|
40
|
+
op: 'status';
|
|
41
|
+
} | {
|
|
42
|
+
op: BrowserOperation;
|
|
43
|
+
url: string;
|
|
44
|
+
timeoutMs: number;
|
|
45
|
+
}, options: {
|
|
46
|
+
timeoutMs: number;
|
|
47
|
+
maxBytes: number;
|
|
48
|
+
}): Promise<BrowserPage | BrowserRefusal | BrowserStatus>;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { createConnection } from 'node:net';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { sha256 } from '../util/hash.js';
|
|
5
|
+
/**
|
|
6
|
+
* The Sponge browser bridge. The Sponge extension in the reader's own browser opens a native-messaging port; the
|
|
7
|
+
* browser starts `host.ts`, which listens on a private local socket. A capture asks over that socket, and the
|
|
8
|
+
* extension loads the page with the reader's own sign-ins. Cookies and passwords never leave the browser: Sponge
|
|
9
|
+
* receives only the page.
|
|
10
|
+
*/
|
|
11
|
+
export const hostName = 'com.hraness.sponge';
|
|
12
|
+
/** Derived from the public key in extension/manifest.json, so an unpacked or store install keeps one identity. */
|
|
13
|
+
export const extensionId = 'pcekakbinokokbcfdpgdljpeoiceoejf';
|
|
14
|
+
/** One bridge per library: a short path, because a Unix socket path is limited to 104 bytes on macOS. */
|
|
15
|
+
export function bridgeSocket(libraryRoot, home = homedir()) {
|
|
16
|
+
return join(home, '.local', 'state', 'sponge', sha256(libraryRoot).slice(0, 16) + '.sock');
|
|
17
|
+
}
|
|
18
|
+
/** Native messaging frames: a 32-bit length in the host's byte order (little-endian on every supported Mac), then JSON. */
|
|
19
|
+
export function encodeFrame(message) {
|
|
20
|
+
const body = Buffer.from(JSON.stringify(message), 'utf8'), frame = Buffer.alloc(4 + body.length);
|
|
21
|
+
frame.writeUInt32LE(body.length, 0);
|
|
22
|
+
body.copy(frame, 4);
|
|
23
|
+
return frame;
|
|
24
|
+
}
|
|
25
|
+
/** Splits complete frames off `buffer`; the rest waits for more bytes. A frame over `maxBytes` is refused. */
|
|
26
|
+
export function decodeFrames(buffer, maxBytes) {
|
|
27
|
+
const messages = [];
|
|
28
|
+
let offset = 0;
|
|
29
|
+
while (buffer.length - offset >= 4) {
|
|
30
|
+
const length = buffer.readUInt32LE(offset);
|
|
31
|
+
if (length > maxBytes)
|
|
32
|
+
throw new Error('native message exceeds its limit');
|
|
33
|
+
if (buffer.length - offset - 4 < length)
|
|
34
|
+
break;
|
|
35
|
+
messages.push(JSON.parse(buffer.subarray(offset + 4, offset + 4 + length).toString('utf8')));
|
|
36
|
+
offset += 4 + length;
|
|
37
|
+
}
|
|
38
|
+
return { messages, rest: buffer.subarray(offset) };
|
|
39
|
+
}
|
|
40
|
+
/** Requests over the socket are one JSON line each way; the bridge closes after its answer. */
|
|
41
|
+
export function askBridge(socket, request, options) {
|
|
42
|
+
return new Promise((resolve) => {
|
|
43
|
+
const chunks = [];
|
|
44
|
+
let bytes = 0, settled = false;
|
|
45
|
+
const done = (value) => {
|
|
46
|
+
if (settled)
|
|
47
|
+
return;
|
|
48
|
+
settled = true;
|
|
49
|
+
clearTimeout(timer);
|
|
50
|
+
connection.destroy();
|
|
51
|
+
resolve(value);
|
|
52
|
+
};
|
|
53
|
+
const connection = createConnection(socket);
|
|
54
|
+
const timer = setTimeout(() => done({ ok: false, code: 'timeout' }), options.timeoutMs);
|
|
55
|
+
connection.on('connect', () => connection.write(JSON.stringify(request) + '\n'));
|
|
56
|
+
connection.on('data', (chunk) => {
|
|
57
|
+
bytes += chunk.length;
|
|
58
|
+
if (bytes > options.maxBytes)
|
|
59
|
+
return done({ ok: false, code: 'too-large' });
|
|
60
|
+
chunks.push(chunk);
|
|
61
|
+
const text = Buffer.concat(chunks).toString('utf8'), end = text.indexOf('\n');
|
|
62
|
+
if (end < 0)
|
|
63
|
+
return;
|
|
64
|
+
try {
|
|
65
|
+
done(answer(JSON.parse(text.slice(0, end))));
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
done({ ok: false, code: 'failed' });
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
connection.on('error', () => done({ ok: false, code: 'offline' }));
|
|
72
|
+
connection.on('close', () => done({ ok: false, code: 'failed' }));
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
const refusals = new Set(['disabled', 'offline', 'failed', 'timeout', 'not-html', 'too-large']);
|
|
76
|
+
/** The bridge relays what the extension says; only these shapes pass. */
|
|
77
|
+
function answer(value) {
|
|
78
|
+
if (value?.ok === true && typeof value.html === 'string' && typeof value.finalUrl === 'string')
|
|
79
|
+
return {
|
|
80
|
+
ok: true,
|
|
81
|
+
html: value.html,
|
|
82
|
+
finalUrl: value.finalUrl,
|
|
83
|
+
...(typeof value.title === 'string' ? { title: value.title } : {}),
|
|
84
|
+
};
|
|
85
|
+
if (value?.ok === true && typeof value.extension === 'string' && typeof value.background === 'boolean')
|
|
86
|
+
return { ok: true, extension: value.extension, background: value.background };
|
|
87
|
+
if (value?.ok === false && refusals.has(value.code))
|
|
88
|
+
return { ok: false, code: value.code, ...(Number.isInteger(value.status) ? { status: value.status } : {}) };
|
|
89
|
+
return { ok: false, code: 'failed' };
|
|
90
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface HostOptions {
|
|
2
|
+
library: string;
|
|
3
|
+
socket: string;
|
|
4
|
+
input: NodeJS.ReadableStream;
|
|
5
|
+
output: NodeJS.WritableStream;
|
|
6
|
+
/** The caller's origin, as the browser passes it; anything but the Sponge extension is refused. */
|
|
7
|
+
origin?: string;
|
|
8
|
+
}
|
|
9
|
+
/** Relays capture requests from the socket to the extension and saves the pages the reader sends. */
|
|
10
|
+
export declare function runHost(options: HostOptions): Promise<void>;
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
// The native-messaging host the browser starts for the Sponge extension. stdout carries only frames to the extension.
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { chmodSync, mkdirSync, realpathSync, rmSync } from 'node:fs';
|
|
4
|
+
import { createServer } from 'node:net';
|
|
5
|
+
import { dirname } from 'node:path';
|
|
6
|
+
import { pathToFileURL } from 'node:url';
|
|
7
|
+
import { decodeFrames, encodeFrame, extensionId } from './bridge.js';
|
|
8
|
+
import { inferKind, articleSource } from '../app/source.js';
|
|
9
|
+
import { pdfSource } from '../pdf/source.js';
|
|
10
|
+
import { openLibrary } from '../store/open.js';
|
|
11
|
+
/** Chrome accepts messages to the extension up to 1 MB and sends up to 64 MiB; Sponge caps a page lower. */
|
|
12
|
+
const maxInboundBytes = 32 * 1024 ** 2;
|
|
13
|
+
/** Relays capture requests from the socket to the extension and saves the pages the reader sends. */
|
|
14
|
+
export function runHost(options) {
|
|
15
|
+
if (options.origin !== undefined && options.origin !== `chrome-extension://${extensionId}/`)
|
|
16
|
+
return Promise.reject(new Error('unexpected caller'));
|
|
17
|
+
const pending = new Map();
|
|
18
|
+
let hello;
|
|
19
|
+
const send = (message) => options.output.write(encodeFrame(message));
|
|
20
|
+
mkdirSync(dirname(options.socket), { recursive: true, mode: 0o700 });
|
|
21
|
+
chmodSync(dirname(options.socket), 0o700);
|
|
22
|
+
// The newest browser to start the host serves the library; a stale socket from a crash is replaced.
|
|
23
|
+
rmSync(options.socket, { force: true });
|
|
24
|
+
const server = createServer((client) => serve(client));
|
|
25
|
+
server.listen(options.socket, () => chmodSync(options.socket, 0o600));
|
|
26
|
+
function serve(client) {
|
|
27
|
+
let buffered = '';
|
|
28
|
+
const reply = (value) => client.end(JSON.stringify(value) + '\n');
|
|
29
|
+
client.setEncoding('utf8');
|
|
30
|
+
client.on('error', () => undefined);
|
|
31
|
+
client.on('data', (chunk) => {
|
|
32
|
+
buffered += chunk;
|
|
33
|
+
if (buffered.length > 64 * 1024)
|
|
34
|
+
return client.destroy();
|
|
35
|
+
const end = buffered.indexOf('\n');
|
|
36
|
+
if (end < 0)
|
|
37
|
+
return;
|
|
38
|
+
let request;
|
|
39
|
+
try {
|
|
40
|
+
request = JSON.parse(buffered.slice(0, end));
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return reply({ ok: false, code: 'failed' });
|
|
44
|
+
}
|
|
45
|
+
if (request?.op === 'status')
|
|
46
|
+
return reply(hello ? { ok: true, ...hello } : { ok: false, code: 'offline' });
|
|
47
|
+
const op = request?.op;
|
|
48
|
+
if ((op !== 'fetch' && op !== 'render') || !web(request.url) || !Number.isInteger(request.timeoutMs))
|
|
49
|
+
return reply({ ok: false, code: 'failed' });
|
|
50
|
+
if (!hello)
|
|
51
|
+
return reply({ ok: false, code: 'offline' });
|
|
52
|
+
if (!hello.background)
|
|
53
|
+
return reply({ ok: false, code: 'disabled' });
|
|
54
|
+
const id = randomUUID(), timeoutMs = Math.min(Math.max(request.timeoutMs, 1000), 120_000), timer = setTimeout(() => finish({ ok: false, code: 'timeout' }), timeoutMs + 5000);
|
|
55
|
+
const finish = (value) => {
|
|
56
|
+
clearTimeout(timer);
|
|
57
|
+
pending.delete(id);
|
|
58
|
+
reply(value);
|
|
59
|
+
};
|
|
60
|
+
pending.set(id, { reply: finish });
|
|
61
|
+
send({ id, op, url: request.url, timeoutMs });
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
async function save(requestId, url) {
|
|
65
|
+
if (typeof requestId !== 'string' || requestId.length > 64)
|
|
66
|
+
return;
|
|
67
|
+
const kind = typeof url === 'string' && web(url) ? inferKind(url) : 'media';
|
|
68
|
+
if (kind === 'media')
|
|
69
|
+
return send({ type: 'saved', requestId, ok: false, code: 'unsupported' });
|
|
70
|
+
try {
|
|
71
|
+
const store = await openLibrary(options.library);
|
|
72
|
+
try {
|
|
73
|
+
const source = (kind === 'pdf' ? pdfSource : articleSource)(url);
|
|
74
|
+
// A page saved from the browser captures through it: the reader is looking at the page they want kept.
|
|
75
|
+
const submission = store.queue.submit({
|
|
76
|
+
kind,
|
|
77
|
+
...source,
|
|
78
|
+
intent: kind === 'article' ? 'browser' : 'save',
|
|
79
|
+
idempotencyKey: randomUUID(),
|
|
80
|
+
});
|
|
81
|
+
send({ type: 'saved', requestId, ok: true, itemId: submission.itemId });
|
|
82
|
+
}
|
|
83
|
+
finally {
|
|
84
|
+
store.close();
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
send({ type: 'saved', requestId, ok: false, code: 'failed' });
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return new Promise((resolve) => {
|
|
92
|
+
let buffer = Buffer.alloc(0);
|
|
93
|
+
options.input.on('data', (chunk) => {
|
|
94
|
+
let decoded;
|
|
95
|
+
try {
|
|
96
|
+
decoded = decodeFrames(Buffer.concat([buffer, chunk]), maxInboundBytes);
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return stop();
|
|
100
|
+
}
|
|
101
|
+
buffer = decoded.rest;
|
|
102
|
+
for (const message of decoded.messages) {
|
|
103
|
+
if (message?.type === 'hello' || message?.type === 'settings')
|
|
104
|
+
hello = {
|
|
105
|
+
extension: String(message.version ?? hello?.extension ?? ''),
|
|
106
|
+
background: message.background !== false,
|
|
107
|
+
};
|
|
108
|
+
else if (message?.type === 'save')
|
|
109
|
+
void save(message.requestId, message.url);
|
|
110
|
+
else if (typeof message?.id === 'string')
|
|
111
|
+
pending.get(message.id)?.reply(message.ok ? message : refusal(message));
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
const stop = () => {
|
|
115
|
+
for (const { reply } of pending.values())
|
|
116
|
+
reply({ ok: false, code: 'offline' });
|
|
117
|
+
server.close();
|
|
118
|
+
rmSync(options.socket, { force: true });
|
|
119
|
+
resolve();
|
|
120
|
+
};
|
|
121
|
+
options.input.on('end', stop);
|
|
122
|
+
options.input.on('error', stop);
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
const web = (value) => {
|
|
126
|
+
if (typeof value !== 'string' || value.length > 16384)
|
|
127
|
+
return false;
|
|
128
|
+
try {
|
|
129
|
+
const url = new URL(value);
|
|
130
|
+
return (url.protocol === 'https:' || url.protocol === 'http:') && !url.username && !url.password;
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
const refusal = (message) => ({
|
|
137
|
+
ok: false,
|
|
138
|
+
code: message.code,
|
|
139
|
+
...(Number.isInteger(message.status) ? { status: message.status } : {}),
|
|
140
|
+
});
|
|
141
|
+
// Started by the wrapper `sponge browser setup` writes: --library ROOT --socket PATH, then the browser's own arguments.
|
|
142
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) {
|
|
143
|
+
const [, , flag1, library, flag2, socket, origin] = process.argv;
|
|
144
|
+
if (flag1 !== '--library' || flag2 !== '--socket' || !library || !socket)
|
|
145
|
+
process.exit(2);
|
|
146
|
+
runHost({ library, socket, origin, input: process.stdin, output: process.stdout }).then(() => process.exit(0), () => process.exit(1));
|
|
147
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export interface BrowserSetupOptions {
|
|
2
|
+
home?: string;
|
|
3
|
+
/** Extra NativeMessagingHosts directories, such as a custom `--user-data-dir`'s. */
|
|
4
|
+
hostsDirs?: string[];
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Registers the native host with each installed browser for one library. The host is a wrapper script, since a
|
|
8
|
+
* manifest names one executable and takes no arguments; it runs this Node and this Sponge's host.
|
|
9
|
+
*/
|
|
10
|
+
export declare function setupBrowser(libraryRoot: string, options?: BrowserSetupOptions): Promise<{
|
|
11
|
+
registered: string[];
|
|
12
|
+
extension: string;
|
|
13
|
+
extensionId: string;
|
|
14
|
+
}>;
|
|
15
|
+
/** Which browsers have the host registered, and whether an extension is connected for this library now. */
|
|
16
|
+
export declare function browserStatus(libraryRoot: string, options?: BrowserSetupOptions): Promise<{
|
|
17
|
+
extension?: string | undefined;
|
|
18
|
+
background?: boolean | undefined;
|
|
19
|
+
registered: string[];
|
|
20
|
+
bound: boolean;
|
|
21
|
+
connected: boolean;
|
|
22
|
+
}>;
|
|
23
|
+
/** Removes the registrations that point at Sponge's host; another program's manifest is left alone. */
|
|
24
|
+
export declare function removeBrowser(options?: BrowserSetupOptions): Promise<{
|
|
25
|
+
removed: string[];
|
|
26
|
+
}>;
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { chmod, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { homedir } from 'node:os';
|
|
4
|
+
import { dirname, join } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import { askBridge, bridgeSocket, extensionId, hostName } from './bridge.js';
|
|
7
|
+
/** Where each Chromium browser on macOS looks for user-level native messaging hosts. */
|
|
8
|
+
const browsers = {
|
|
9
|
+
Chrome: 'Google/Chrome',
|
|
10
|
+
'Chrome Beta': 'Google/Chrome Beta',
|
|
11
|
+
'Chrome Canary': 'Google/Chrome Canary',
|
|
12
|
+
Chromium: 'Chromium',
|
|
13
|
+
Brave: 'BraveSoftware/Brave-Browser',
|
|
14
|
+
Edge: 'Microsoft Edge',
|
|
15
|
+
Arc: 'Arc/User Data',
|
|
16
|
+
Vivaldi: 'Vivaldi',
|
|
17
|
+
};
|
|
18
|
+
const manifestName = `${hostName}.json`;
|
|
19
|
+
/** The installed browsers' host directories, plus any the reader named. */
|
|
20
|
+
function hostDirectories(options) {
|
|
21
|
+
const support = join(options.home ?? homedir(), 'Library', 'Application Support');
|
|
22
|
+
return [
|
|
23
|
+
...Object.entries(browsers)
|
|
24
|
+
.filter(([, data]) => existsSync(join(support, data)))
|
|
25
|
+
.map(([browser, data]) => ({ browser, dir: join(support, data, 'NativeMessagingHosts') })),
|
|
26
|
+
...(options.hostsDirs ?? []).map((dir) => ({ browser: 'custom', dir })),
|
|
27
|
+
];
|
|
28
|
+
}
|
|
29
|
+
const wrapperPath = (home) => join(home, '.local', 'state', 'sponge', 'browser-host');
|
|
30
|
+
/** The unpacked extension that ships with this Sponge. */
|
|
31
|
+
function extensionDirectory() {
|
|
32
|
+
return fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? '../../extension/' : '../../../extension/', import.meta.url));
|
|
33
|
+
}
|
|
34
|
+
const quote = (value) => `'${value.replace(/'/g, `'\\''`)}'`;
|
|
35
|
+
/**
|
|
36
|
+
* Registers the native host with each installed browser for one library. The host is a wrapper script, since a
|
|
37
|
+
* manifest names one executable and takes no arguments; it runs this Node and this Sponge's host.
|
|
38
|
+
*/
|
|
39
|
+
export async function setupBrowser(libraryRoot, options = {}) {
|
|
40
|
+
const home = options.home ?? homedir(), wrapper = wrapperPath(home), host = fileURLToPath(new URL('./host.js', import.meta.url));
|
|
41
|
+
await mkdir(dirname(wrapper), { recursive: true, mode: 0o700 });
|
|
42
|
+
await writeFile(wrapper, `#!/bin/sh\nexec ${quote(process.execPath)} ${quote(host)} --library ${quote(libraryRoot)} --socket ${quote(bridgeSocket(libraryRoot, home))} "$@"\n`, { mode: 0o700 });
|
|
43
|
+
await chmod(wrapper, 0o700);
|
|
44
|
+
const manifest = JSON.stringify({
|
|
45
|
+
name: hostName,
|
|
46
|
+
description: 'Sponge: save pages with your own sign-ins',
|
|
47
|
+
path: wrapper,
|
|
48
|
+
type: 'stdio',
|
|
49
|
+
allowed_origins: [`chrome-extension://${extensionId}/`],
|
|
50
|
+
}, null, 2);
|
|
51
|
+
const registered = [];
|
|
52
|
+
for (const { browser, dir } of hostDirectories(options)) {
|
|
53
|
+
await mkdir(dir, { recursive: true });
|
|
54
|
+
await writeFile(join(dir, manifestName), manifest + '\n', { mode: 0o644 });
|
|
55
|
+
registered.push(browser);
|
|
56
|
+
}
|
|
57
|
+
return { registered, extension: extensionDirectory(), extensionId };
|
|
58
|
+
}
|
|
59
|
+
/** Which browsers have the host registered, and whether an extension is connected for this library now. */
|
|
60
|
+
export async function browserStatus(libraryRoot, options = {}) {
|
|
61
|
+
const home = options.home ?? homedir(), wrapper = wrapperPath(home), registered = [];
|
|
62
|
+
for (const { browser, dir } of hostDirectories(options)) {
|
|
63
|
+
try {
|
|
64
|
+
if (JSON.parse(await readFile(join(dir, manifestName), 'utf8')).path === wrapper)
|
|
65
|
+
registered.push(browser);
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
// Not registered with this browser.
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
let bound = false;
|
|
72
|
+
try {
|
|
73
|
+
bound = (await readFile(wrapper, 'utf8')).includes(quote(bridgeSocket(libraryRoot, home)));
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
bound = false;
|
|
77
|
+
}
|
|
78
|
+
const answer = await askBridge(bridgeSocket(libraryRoot, home), { op: 'status' }, { timeoutMs: 3000, maxBytes: 4096 });
|
|
79
|
+
return {
|
|
80
|
+
registered,
|
|
81
|
+
bound,
|
|
82
|
+
connected: answer.ok,
|
|
83
|
+
...(answer.ok && 'background' in answer ? { extension: answer.extension, background: answer.background } : {}),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
/** Removes the registrations that point at Sponge's host; another program's manifest is left alone. */
|
|
87
|
+
export async function removeBrowser(options = {}) {
|
|
88
|
+
const home = options.home ?? homedir(), wrapper = wrapperPath(home), removed = [];
|
|
89
|
+
for (const { browser, dir } of hostDirectories(options)) {
|
|
90
|
+
const file = join(dir, manifestName);
|
|
91
|
+
try {
|
|
92
|
+
if (JSON.parse(await readFile(file, 'utf8')).path !== wrapper)
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
await rm(file, { force: true });
|
|
99
|
+
removed.push(browser);
|
|
100
|
+
}
|
|
101
|
+
await rm(wrapper, { force: true });
|
|
102
|
+
return { removed };
|
|
103
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export type CaptureScope = 'auto' | 'page' | 'thread' | 'comments';
|
|
2
|
-
export type AcquisitionMethod = 'http' | 'hacker-news-api' | 'bluesky-api' | 'reddit-json' | 'browser-cdp';
|
|
2
|
+
export type AcquisitionMethod = 'http' | 'hacker-news-api' | 'bluesky-api' | 'reddit-json' | 'browser-cdp' | 'browser-fetch' | 'browser-render';
|
|
3
3
|
export type AcquiredPage = {
|
|
4
4
|
readonly body: string;
|
|
5
5
|
readonly contentType: string | null;
|