@sovovs/bycli 2.1.40 → 2.1.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli-manifest.json +10 -2
- package/clis/weixin/_wechat/article-artifact.js +55 -0
- package/clis/weixin/_wechat/article-identity.js +27 -0
- package/clis/weixin/_wechat/publish-analysis.js +8 -4
- package/clis/weixin/_wechat/publish-download.js +3 -1
- package/clis/weixin/download-publish-data.js +20 -0
- package/clis/weixin/download.js +15 -2
- package/dist/src/adapter-coordination.d.ts +26 -0
- package/dist/src/adapter-coordination.js +183 -0
- package/dist/src/adapter-coordination.test.d.ts +1 -0
- package/dist/src/adapter-execution-context.d.ts +6 -0
- package/dist/src/adapter-execution-context.js +8 -0
- package/dist/src/adapter-scheduler.d.ts +86 -0
- package/dist/src/adapter-scheduler.js +349 -0
- package/dist/src/adapter-scheduler.test.d.ts +1 -0
- package/dist/src/browser/daemon-client.d.ts +11 -0
- package/dist/src/browser/daemon-client.js +53 -1
- package/dist/src/browser/extension-capabilities.d.ts +1 -0
- package/dist/src/browser/extension-capabilities.js +18 -5
- package/dist/src/browser/page.d.ts +2 -1
- package/dist/src/browser/page.js +3 -0
- package/dist/src/build-manifest.js +1 -0
- package/dist/src/cli-argv-preprocess.d.ts +3 -0
- package/dist/src/cli-argv-preprocess.js +4 -0
- package/dist/src/commanderAdapter.js +11 -0
- package/dist/src/daemon.js +118 -0
- package/dist/src/discovery.js +1 -0
- package/dist/src/download/article-download.d.ts +2 -0
- package/dist/src/download/article-download.js +30 -4
- package/dist/src/errors.d.ts +3 -0
- package/dist/src/errors.js +5 -0
- package/dist/src/execution.d.ts +2 -0
- package/dist/src/execution.js +172 -105
- package/dist/src/help.d.ts +1 -0
- package/dist/src/help.js +40 -0
- package/dist/src/manifest-types.d.ts +4 -0
- package/dist/src/registry.d.ts +6 -0
- package/dist/src/registry.js +23 -0
- package/dist/src/serialization.d.ts +1 -0
- package/dist/src/serialization.js +1 -0
- package/dist/src/types.d.ts +2 -0
- package/package.json +3 -2
package/cli-manifest.json
CHANGED
|
@@ -29080,7 +29080,11 @@
|
|
|
29080
29080
|
"type": "js",
|
|
29081
29081
|
"modulePath": "weixin/download.js",
|
|
29082
29082
|
"sourceFile": "weixin/download.js",
|
|
29083
|
-
"navigateBefore": "https://mp.weixin.qq.com"
|
|
29083
|
+
"navigateBefore": "https://mp.weixin.qq.com",
|
|
29084
|
+
"adapterConcurrency": {
|
|
29085
|
+
"isolatedTabs": true,
|
|
29086
|
+
"maxParallel": 3
|
|
29087
|
+
}
|
|
29084
29088
|
},
|
|
29085
29089
|
{
|
|
29086
29090
|
"site": "weixin",
|
|
@@ -29159,7 +29163,11 @@
|
|
|
29159
29163
|
"type": "js",
|
|
29160
29164
|
"modulePath": "weixin/download-publish-data.js",
|
|
29161
29165
|
"sourceFile": "weixin/download-publish-data.js",
|
|
29162
|
-
"navigateBefore": false
|
|
29166
|
+
"navigateBefore": false,
|
|
29167
|
+
"adapterConcurrency": {
|
|
29168
|
+
"isolatedTabs": true,
|
|
29169
|
+
"maxParallel": 3
|
|
29170
|
+
}
|
|
29163
29171
|
},
|
|
29164
29172
|
{
|
|
29165
29173
|
"site": "weixin",
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { constants } from 'node:fs';
|
|
2
|
+
import { access, readFile, realpath, stat } from 'node:fs/promises';
|
|
3
|
+
import { dirname, extname, relative, resolve } from 'node:path';
|
|
4
|
+
import { CommandExecutionError } from '@sovovs/bycli/errors';
|
|
5
|
+
|
|
6
|
+
function isContained(parent, child) {
|
|
7
|
+
const rel = relative(parent, child);
|
|
8
|
+
return rel === '' || (!rel.startsWith('..') && !rel.startsWith('/') && !rel.startsWith('\\'));
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
async function validateRegularNonEmptyFile(path, label) {
|
|
12
|
+
await access(path, constants.R_OK);
|
|
13
|
+
const info = await stat(path);
|
|
14
|
+
if (!info.isFile() || info.size <= 0) throw new Error(`${label} is empty or not a regular file`);
|
|
15
|
+
return realpath(path);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function localMarkdownImageTargets(markdown) {
|
|
19
|
+
const targets = [];
|
|
20
|
+
for (const match of markdown.matchAll(/!\[[^\]]*\]\(([^)]+)\)/g)) {
|
|
21
|
+
let target = match[1].trim();
|
|
22
|
+
if (target.startsWith('<') && target.endsWith('>')) target = target.slice(1, -1);
|
|
23
|
+
target = target.split(/\s+["']/u, 1)[0];
|
|
24
|
+
if (!target || /^(?:https?:|data:|#|\/\/)/iu.test(target)) continue;
|
|
25
|
+
targets.push(decodeURI(target));
|
|
26
|
+
}
|
|
27
|
+
return targets;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function validateDownloadedArticleRows(rows, outputDir) {
|
|
31
|
+
try {
|
|
32
|
+
const successfulRows = rows.filter(row => row && String(row.status).toLowerCase() === 'success');
|
|
33
|
+
if (successfulRows.length === 0) return rows;
|
|
34
|
+
const resolvedOutput = await realpath(resolve(outputDir));
|
|
35
|
+
for (const row of successfulRows) {
|
|
36
|
+
if (typeof row.saved !== 'string' || extname(row.saved).toLowerCase() !== '.md') {
|
|
37
|
+
throw new Error('successful article row returned no Markdown path');
|
|
38
|
+
}
|
|
39
|
+
const saved = await validateRegularNonEmptyFile(resolve(row.saved), 'saved Markdown');
|
|
40
|
+
if (!isContained(resolvedOutput, saved)) throw new Error('saved Markdown escaped the output directory');
|
|
41
|
+
const articleDir = dirname(saved);
|
|
42
|
+
const markdown = await readFile(saved, 'utf8');
|
|
43
|
+
for (const target of localMarkdownImageTargets(markdown)) {
|
|
44
|
+
const image = await validateRegularNonEmptyFile(resolve(articleDir, target), 'local Markdown image');
|
|
45
|
+
if (!isContained(articleDir, image)) throw new Error('local Markdown image escaped the article directory');
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return rows;
|
|
49
|
+
} catch (error) {
|
|
50
|
+
if (error instanceof CommandExecutionError) throw error;
|
|
51
|
+
throw new CommandExecutionError(
|
|
52
|
+
`Downloaded Weixin article artifact validation failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
export function hashResourceValue(value) {
|
|
4
|
+
return createHash('sha256').update(String(value)).digest('hex');
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export function canonicalWechatArticleIdentity(rawUrl) {
|
|
8
|
+
const url = new URL(rawUrl);
|
|
9
|
+
url.hash = '';
|
|
10
|
+
const host = url.hostname.toLowerCase();
|
|
11
|
+
let canonical;
|
|
12
|
+
if (url.pathname.startsWith('/s/')) {
|
|
13
|
+
canonical = `${host}${url.pathname}`;
|
|
14
|
+
} else {
|
|
15
|
+
const tupleNames = ['__biz', 'mid', 'idx', 'sn'];
|
|
16
|
+
const hasTuple = tupleNames.every(name => url.searchParams.has(name));
|
|
17
|
+
if (url.pathname === '/s' && hasTuple) {
|
|
18
|
+
canonical = `${host}/s?${tupleNames.map(name => `${name}=${url.searchParams.get(name)}`).join('&')}`;
|
|
19
|
+
} else {
|
|
20
|
+
const sorted = [...url.searchParams.entries()]
|
|
21
|
+
.sort(([leftKey, leftValue], [rightKey, rightValue]) => leftKey.localeCompare(rightKey) || leftValue.localeCompare(rightValue));
|
|
22
|
+
const query = new URLSearchParams(sorted).toString();
|
|
23
|
+
canonical = `${host}${url.pathname}${query ? `?${query}` : ''}`;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return hashResourceValue(canonical);
|
|
27
|
+
}
|
|
@@ -427,7 +427,7 @@ function safeFilename(title) {
|
|
|
427
427
|
return `${name}.md`;
|
|
428
428
|
}
|
|
429
429
|
|
|
430
|
-
async function publishMarkdown(outputDir, filename, content) {
|
|
430
|
+
async function publishMarkdown(outputDir, filename, content, beforePublish) {
|
|
431
431
|
await mkdir(outputDir, { recursive: true });
|
|
432
432
|
const temporary = resolve(outputDir, `.bycli-publish-analysis-${randomUUID()}.tmp`);
|
|
433
433
|
await writeFile(temporary, content, { encoding: 'utf8', flag: 'wx' });
|
|
@@ -436,13 +436,17 @@ async function publishMarkdown(outputDir, filename, content) {
|
|
|
436
436
|
const stem = filename.slice(0, -extension.length);
|
|
437
437
|
for (let index = 0; index <= 9999; index += 1) {
|
|
438
438
|
const path = resolve(outputDir, index === 0 ? filename : `${stem}-${index}${extension}`);
|
|
439
|
-
try {
|
|
439
|
+
try {
|
|
440
|
+
await beforePublish?.();
|
|
441
|
+
await link(temporary, path);
|
|
442
|
+
return path;
|
|
443
|
+
} catch (error) { if (error?.code !== 'EEXIST') throw error; }
|
|
440
444
|
}
|
|
441
445
|
throw new CommandExecutionError('WeChat publish analysis could not allocate a report filename');
|
|
442
446
|
} finally { await unlink(temporary).catch(() => {}); }
|
|
443
447
|
}
|
|
444
448
|
|
|
445
|
-
export async function collectPublishAnalysis(page, { detailUrl, title, publishedAt, outputDir }) {
|
|
449
|
+
export async function collectPublishAnalysis(page, { detailUrl, title, publishedAt, outputDir, beforePublish }) {
|
|
446
450
|
if (typeof page?.goto !== 'function' || typeof page?.readNetworkCapture !== 'function') {
|
|
447
451
|
throw new CommandExecutionError('WeChat publish analysis requires browser network capture support');
|
|
448
452
|
}
|
|
@@ -480,7 +484,7 @@ export async function collectPublishAnalysis(page, { detailUrl, title, published
|
|
|
480
484
|
}
|
|
481
485
|
if (Object.keys(data).length === 0) throw new CommandExecutionError('WeChat publish analysis returned no readable analysis data');
|
|
482
486
|
const content = formatAnalysisMarkdown({ title, publishedAt, data });
|
|
483
|
-
const path = await publishMarkdown(resolve(outputDir), safeFilename(title), content);
|
|
487
|
+
const path = await publishMarkdown(resolve(outputDir), safeFilename(title), content, beforePublish);
|
|
484
488
|
const info = await stat(path);
|
|
485
489
|
return { status: 'saved', path, size: info.size, metrics };
|
|
486
490
|
}
|
|
@@ -37,7 +37,7 @@ function safeFilename(title) {
|
|
|
37
37
|
return name;
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
async function publishExclusively(source, outputDir, filename) {
|
|
40
|
+
async function publishExclusively(source, outputDir, filename, beforePublish) {
|
|
41
41
|
const extension = extname(filename);
|
|
42
42
|
const stem = filename.slice(0, -extension.length);
|
|
43
43
|
const temporary = resolve(outputDir, `.bycli-publish-data-${randomUUID()}.tmp`);
|
|
@@ -61,6 +61,7 @@ async function publishExclusively(source, outputDir, filename) {
|
|
|
61
61
|
for (let index = 0; index <= 9999; index += 1) {
|
|
62
62
|
const candidate = resolve(outputDir, index === 0 ? filename : `${stem}-${index}${extension}`);
|
|
63
63
|
try {
|
|
64
|
+
await beforePublish?.();
|
|
64
65
|
await link(temporary, candidate);
|
|
65
66
|
return candidate;
|
|
66
67
|
} catch (error) {
|
|
@@ -151,6 +152,7 @@ export async function downloadPublishData(page, options) {
|
|
|
151
152
|
downloaded.filename,
|
|
152
153
|
outputDir,
|
|
153
154
|
safeFilename(options.title),
|
|
155
|
+
options.beforePublish,
|
|
154
156
|
);
|
|
155
157
|
try {
|
|
156
158
|
await unlink(downloaded.filename);
|
|
@@ -3,6 +3,7 @@ import { access, stat } from 'node:fs/promises';
|
|
|
3
3
|
import { extname, resolve } from 'node:path';
|
|
4
4
|
import { ArgumentError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
5
5
|
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
6
|
+
import { assertCurrentAdapterLease, withAdapterResourceLocks } from '@sovovs/bycli/adapter-coordination';
|
|
6
7
|
import { resolveBrowserCredentials } from './_wechat/auth-session.js';
|
|
7
8
|
import { buildSecretSet, redactText } from './_wechat/redact.js';
|
|
8
9
|
import { collectPublishAnalysis } from './_wechat/publish-analysis.js';
|
|
@@ -15,6 +16,7 @@ import {
|
|
|
15
16
|
validatePublishDate,
|
|
16
17
|
validatePublishedQuery,
|
|
17
18
|
} from './_wechat/publish-records.js';
|
|
19
|
+
import { canonicalWechatArticleIdentity, hashResourceValue } from './_wechat/article-identity.js';
|
|
18
20
|
|
|
19
21
|
const METRIC_COLUMNS = [
|
|
20
22
|
'readUsers', 'avgReadMinutes', 'finishedReadRatio', 'newFollowers', 'listenUsers',
|
|
@@ -35,6 +37,13 @@ function sanitizedError(error, secrets, fallback) {
|
|
|
35
37
|
.replace(/https?:\/\/mp\.weixin\.qq\.com\/\S*/giu, '[REDACTED]');
|
|
36
38
|
}
|
|
37
39
|
|
|
40
|
+
function rethrowStopError(error) {
|
|
41
|
+
const code = error && typeof error === 'object' ? error.code : undefined;
|
|
42
|
+
if (code === 'AUTH_REQUIRED' || code === 'RATE_LIMITED' || code === 'CAPTCHA' || code === 'MFA_REQUIRED' || code === 'ADAPTER_LEASE_LOST') {
|
|
43
|
+
throw error;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
38
47
|
async function validateArtifact(result, { label, expectedStatus, expectedExtension }) {
|
|
39
48
|
if (!result || result.status !== expectedStatus) {
|
|
40
49
|
throw new CommandExecutionError(`${label} returned an invalid status`);
|
|
@@ -71,6 +80,7 @@ export const downloadPublishDataCommand = cli({
|
|
|
71
80
|
strategy: Strategy.INTERCEPT,
|
|
72
81
|
browser: true,
|
|
73
82
|
navigateBefore: false,
|
|
83
|
+
adapterConcurrency: { isolatedTabs: true, maxParallel: 3 },
|
|
74
84
|
args: [
|
|
75
85
|
{ name: 'query', positional: true, required: true, help: 'Exact article URL or title text' },
|
|
76
86
|
{ name: 'date', help: 'Optional publication date in YYYY-MM-DD' },
|
|
@@ -104,9 +114,16 @@ export const downloadPublishDataCommand = cli({
|
|
|
104
114
|
title: record.title,
|
|
105
115
|
outputDir,
|
|
106
116
|
timeoutSeconds,
|
|
117
|
+
beforePublish: assertCurrentAdapterLease,
|
|
107
118
|
};
|
|
108
119
|
const secrets = buildSecretSet({ token, cookie });
|
|
109
120
|
|
|
121
|
+
return withAdapterResourceLocks([
|
|
122
|
+
`article:${canonicalWechatArticleIdentity(record.url)}`,
|
|
123
|
+
`data:${hashResourceValue(`${record.msgid}:${record.publishDate ?? record.publishedAt}`)}`,
|
|
124
|
+
`output:${hashResourceValue(resolve(outputDir))}`,
|
|
125
|
+
], async () => {
|
|
126
|
+
|
|
110
127
|
let dataResult = null;
|
|
111
128
|
let markdownResult = null;
|
|
112
129
|
const errors = [];
|
|
@@ -118,6 +135,7 @@ export const downloadPublishDataCommand = cli({
|
|
|
118
135
|
expectedExtension: '.xls',
|
|
119
136
|
});
|
|
120
137
|
} catch (error) {
|
|
138
|
+
rethrowStopError(error);
|
|
121
139
|
errors.push(`Excel download failed: ${sanitizedError(error, secrets, 'Excel download failed')}`);
|
|
122
140
|
}
|
|
123
141
|
try {
|
|
@@ -131,6 +149,7 @@ export const downloadPublishDataCommand = cli({
|
|
|
131
149
|
expectedExtension: '.md',
|
|
132
150
|
});
|
|
133
151
|
} catch (error) {
|
|
152
|
+
rethrowStopError(error);
|
|
134
153
|
errors.push(`Markdown analysis failed: ${sanitizedError(error, secrets, 'Markdown analysis failed')}`);
|
|
135
154
|
}
|
|
136
155
|
|
|
@@ -175,5 +194,6 @@ export const downloadPublishDataCommand = cli({
|
|
|
175
194
|
detailLikes: metrics?.likes ?? null,
|
|
176
195
|
detailComments: metrics?.comments ?? null,
|
|
177
196
|
}];
|
|
197
|
+
});
|
|
178
198
|
},
|
|
179
199
|
});
|
package/clis/weixin/download.js
CHANGED
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
10
10
|
import { downloadArticle } from '@sovovs/bycli/download/article-download';
|
|
11
11
|
import { AuthRequiredError } from '@sovovs/bycli/errors';
|
|
12
|
+
import { assertCurrentAdapterLease, withAdapterResourceLocks } from '@sovovs/bycli/adapter-coordination';
|
|
13
|
+
import { resolve } from 'node:path';
|
|
12
14
|
import { buildExtractWechatArticleContentJs } from './_wechat/article-content.js';
|
|
13
15
|
import {
|
|
14
16
|
isTrustedSogouRedirectUrl,
|
|
@@ -16,6 +18,8 @@ import {
|
|
|
16
18
|
normalizeWechatUrl,
|
|
17
19
|
resolveWechatArticleUrl,
|
|
18
20
|
} from './_wechat/article-link.js';
|
|
21
|
+
import { canonicalWechatArticleIdentity, hashResourceValue } from './_wechat/article-identity.js';
|
|
22
|
+
import { validateDownloadedArticleRows } from './_wechat/article-artifact.js';
|
|
19
23
|
export { extractWechatArticleContent } from './_wechat/article-content.js';
|
|
20
24
|
export {
|
|
21
25
|
isTrustedSogouRedirectUrl,
|
|
@@ -152,6 +156,7 @@ cli({
|
|
|
152
156
|
description: '下载微信公众号文章为 Markdown 格式',
|
|
153
157
|
domain: 'mp.weixin.qq.com',
|
|
154
158
|
strategy: Strategy.COOKIE,
|
|
159
|
+
adapterConcurrency: { isolatedTabs: true, maxParallel: 3 },
|
|
155
160
|
args: [
|
|
156
161
|
{ name: 'url', required: true, help: 'WeChat article URL (mp.weixin.qq.com/s/xxx)' },
|
|
157
162
|
{ name: 'output', default: './weixin-articles', help: 'Output directory' },
|
|
@@ -160,6 +165,11 @@ cli({
|
|
|
160
165
|
columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved', 'source_url', 'resolved_url'],
|
|
161
166
|
func: async (page, kwargs) => {
|
|
162
167
|
const { sourceUrl, resolvedUrl, alreadyNavigated } = await resolveWechatArticleUrl(page, kwargs.url);
|
|
168
|
+
const outputDir = resolve(kwargs.output ?? './weixin-articles');
|
|
169
|
+
return withAdapterResourceLocks([
|
|
170
|
+
`article:${canonicalWechatArticleIdentity(resolvedUrl)}`,
|
|
171
|
+
`output:${hashResourceValue(outputDir)}`,
|
|
172
|
+
], async () => {
|
|
163
173
|
// Navigate and wait for content to load. Sogou resolution already lands on the article.
|
|
164
174
|
if (!alreadyNavigated)
|
|
165
175
|
await page.goto(resolvedUrl);
|
|
@@ -235,7 +245,7 @@ cli({
|
|
|
235
245
|
codeBlocks: data?.codeBlocks,
|
|
236
246
|
imageUrls: data?.imageUrls,
|
|
237
247
|
}, {
|
|
238
|
-
output:
|
|
248
|
+
output: outputDir,
|
|
239
249
|
downloadImages: kwargs['download-images'],
|
|
240
250
|
imageHeaders: { Referer: 'https://mp.weixin.qq.com/' },
|
|
241
251
|
frontmatterLabels: { author: '公众号' },
|
|
@@ -244,7 +254,10 @@ cli({
|
|
|
244
254
|
return m ? m[1] : 'png';
|
|
245
255
|
},
|
|
246
256
|
secureMarkdown: true,
|
|
257
|
+
beforePublish: assertCurrentAdapterLease,
|
|
258
|
+
});
|
|
259
|
+
const validatedRows = await validateDownloadedArticleRows(rows, outputDir);
|
|
260
|
+
return validatedRows.map(row => ({ ...row, source_url: sourceUrl, resolved_url: resolvedUrl }));
|
|
247
261
|
});
|
|
248
|
-
return rows.map(row => ({ ...row, source_url: sourceUrl, resolved_url: resolvedUrl }));
|
|
249
262
|
},
|
|
250
263
|
});
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { AdapterLease, AdapterLeaseRelease, AdapterLeaseRequest, AdapterResourceGrant } from './adapter-scheduler.js';
|
|
2
|
+
export interface AdapterCoordinationDependencies {
|
|
3
|
+
acquire?: (request: AdapterLeaseRequest) => Promise<AdapterLease>;
|
|
4
|
+
heartbeat?: (lease: AdapterLease) => Promise<AdapterLease>;
|
|
5
|
+
release?: (release: AdapterLeaseRelease) => Promise<boolean>;
|
|
6
|
+
heartbeatIntervalMs?: number;
|
|
7
|
+
onLeaseLost?: () => Promise<void>;
|
|
8
|
+
warn?: (message: string) => void;
|
|
9
|
+
}
|
|
10
|
+
export declare function getCurrentAdapterLease(): AdapterLease | undefined;
|
|
11
|
+
/**
|
|
12
|
+
* Renew the active lease immediately before an irreversible local publication.
|
|
13
|
+
* A restarted daemon or reclaimed lease rejects this fencing check.
|
|
14
|
+
*/
|
|
15
|
+
export declare function assertCurrentAdapterLease(dependencies?: {
|
|
16
|
+
heartbeat?: (lease: AdapterLease) => Promise<AdapterLease>;
|
|
17
|
+
}): Promise<void>;
|
|
18
|
+
export interface AdapterResourceDependencies {
|
|
19
|
+
acquire?: (lease: AdapterLease, keys: string[], timeoutMs: number) => Promise<AdapterResourceGrant>;
|
|
20
|
+
release?: (lease: AdapterLease, grantId: string) => Promise<boolean>;
|
|
21
|
+
timeoutMs?: number;
|
|
22
|
+
warn?: (message: string) => void;
|
|
23
|
+
}
|
|
24
|
+
export declare function settleAdapterOperationAfterTimeout<T>(operation: Promise<T>, timeoutMs: number, timeoutError: Error, stop: () => Promise<void>): Promise<T>;
|
|
25
|
+
export declare function withAdapterResourceLocks<T>(keys: string[], operation: () => Promise<T>, dependencies?: AdapterResourceDependencies): Promise<T>;
|
|
26
|
+
export declare function withAdapterCommandLease<T>(request: AdapterLeaseRequest, operation: () => Promise<T>, dependencies?: AdapterCoordinationDependencies): Promise<T>;
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { acquireAdapterLease, heartbeatAdapterLease, releaseAdapterLease, acquireAdapterResources, releaseAdapterResources, } from './browser/daemon-client.js';
|
|
2
|
+
import { log } from './logger.js';
|
|
3
|
+
import { AdapterCoordinationError } from './errors.js';
|
|
4
|
+
import { getAdapterExecutionContext, runWithAdapterExecutionContext, } from './adapter-execution-context.js';
|
|
5
|
+
export function getCurrentAdapterLease() {
|
|
6
|
+
return getAdapterExecutionContext()?.lease;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Renew the active lease immediately before an irreversible local publication.
|
|
10
|
+
* A restarted daemon or reclaimed lease rejects this fencing check.
|
|
11
|
+
*/
|
|
12
|
+
export async function assertCurrentAdapterLease(dependencies = {}) {
|
|
13
|
+
const context = getAdapterExecutionContext();
|
|
14
|
+
if (!context)
|
|
15
|
+
return;
|
|
16
|
+
try {
|
|
17
|
+
context.lease = await (dependencies.heartbeat ?? heartbeatAdapterLease)(context.lease);
|
|
18
|
+
}
|
|
19
|
+
catch (error) {
|
|
20
|
+
if (error instanceof AdapterCoordinationError && error.code === 'ADAPTER_LEASE_LOST')
|
|
21
|
+
throw error;
|
|
22
|
+
throw new AdapterCoordinationError('ADAPTER_LEASE_LOST', 'Adapter lease fencing failed before artifact publication.', true);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
export async function settleAdapterOperationAfterTimeout(operation, timeoutMs, timeoutError, stop) {
|
|
26
|
+
const outcome = operation.then(value => ({ kind: 'value', value }), error => ({ kind: 'error', error }));
|
|
27
|
+
let timeout;
|
|
28
|
+
const first = await Promise.race([
|
|
29
|
+
outcome,
|
|
30
|
+
new Promise(resolve => {
|
|
31
|
+
timeout = setTimeout(() => resolve({ kind: 'timeout' }), timeoutMs);
|
|
32
|
+
}),
|
|
33
|
+
]);
|
|
34
|
+
if (first.kind !== 'timeout') {
|
|
35
|
+
if (timeout)
|
|
36
|
+
clearTimeout(timeout);
|
|
37
|
+
if (first.kind === 'error')
|
|
38
|
+
throw first.error;
|
|
39
|
+
return first.value;
|
|
40
|
+
}
|
|
41
|
+
try {
|
|
42
|
+
await stop();
|
|
43
|
+
}
|
|
44
|
+
finally {
|
|
45
|
+
await outcome;
|
|
46
|
+
}
|
|
47
|
+
throw timeoutError;
|
|
48
|
+
}
|
|
49
|
+
export async function withAdapterResourceLocks(keys, operation, dependencies = {}) {
|
|
50
|
+
const lease = getCurrentAdapterLease();
|
|
51
|
+
if (!lease)
|
|
52
|
+
return operation();
|
|
53
|
+
const acquire = dependencies.acquire ?? acquireAdapterResources;
|
|
54
|
+
const release = dependencies.release ?? releaseAdapterResources;
|
|
55
|
+
const warn = dependencies.warn ?? ((message) => log.warn(message));
|
|
56
|
+
const scopedKeys = keys.map(key => key.startsWith('article:') || key.startsWith('data:')
|
|
57
|
+
? `profile:${lease.contextId}:${key}`
|
|
58
|
+
: key);
|
|
59
|
+
const grant = await acquire(lease, scopedKeys, dependencies.timeoutMs ?? 300_000);
|
|
60
|
+
try {
|
|
61
|
+
return await operation();
|
|
62
|
+
}
|
|
63
|
+
finally {
|
|
64
|
+
let released = false;
|
|
65
|
+
let lastError;
|
|
66
|
+
for (let attempt = 0; attempt < 2 && !released; attempt++) {
|
|
67
|
+
try {
|
|
68
|
+
await release(lease, grant.grantId);
|
|
69
|
+
released = true;
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
lastError = error;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
if (!released) {
|
|
76
|
+
warn(`Adapter resource release acknowledgement failed for ${grant.grantId}: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
export async function withAdapterCommandLease(request, operation, dependencies = {}) {
|
|
81
|
+
const acquire = dependencies.acquire ?? acquireAdapterLease;
|
|
82
|
+
const heartbeat = dependencies.heartbeat ?? heartbeatAdapterLease;
|
|
83
|
+
const release = dependencies.release ?? releaseAdapterLease;
|
|
84
|
+
const warn = dependencies.warn ?? ((message) => log.warn(message));
|
|
85
|
+
const heartbeatIntervalMs = dependencies.heartbeatIntervalMs ?? 10_000;
|
|
86
|
+
const context = { lease: await acquire(request) };
|
|
87
|
+
let releaseReason = 'error';
|
|
88
|
+
let leaseLost;
|
|
89
|
+
let stopAfterLeaseLoss;
|
|
90
|
+
let heartbeatTask;
|
|
91
|
+
const heartbeatTimer = setInterval(() => {
|
|
92
|
+
if (heartbeatTask || leaseLost)
|
|
93
|
+
return;
|
|
94
|
+
heartbeatTask = heartbeat(context.lease)
|
|
95
|
+
.then(next => { context.lease = next; })
|
|
96
|
+
.catch(error => {
|
|
97
|
+
const code = error && typeof error === 'object' ? error.code : undefined;
|
|
98
|
+
if (code === 'ADAPTER_LEASE_LOST') {
|
|
99
|
+
leaseLost = error;
|
|
100
|
+
stopAfterLeaseLoss = (dependencies.onLeaseLost?.() ?? Promise.resolve()).catch(stopError => {
|
|
101
|
+
warn(`Adapter operation stop failed after lease loss for ${context.lease.requestId}: ${stopError instanceof Error ? stopError.message : String(stopError)}`);
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
105
|
+
warn(`Adapter lease heartbeat failed for ${context.lease.requestId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
106
|
+
}
|
|
107
|
+
})
|
|
108
|
+
.finally(() => { heartbeatTask = undefined; });
|
|
109
|
+
}, heartbeatIntervalMs);
|
|
110
|
+
heartbeatTimer.unref?.();
|
|
111
|
+
try {
|
|
112
|
+
let result;
|
|
113
|
+
try {
|
|
114
|
+
result = await runWithAdapterExecutionContext(context, operation);
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
if (heartbeatTask)
|
|
118
|
+
await heartbeatTask;
|
|
119
|
+
if (stopAfterLeaseLoss)
|
|
120
|
+
await stopAfterLeaseLoss;
|
|
121
|
+
if (leaseLost)
|
|
122
|
+
throw leaseLost;
|
|
123
|
+
throw error;
|
|
124
|
+
}
|
|
125
|
+
if (heartbeatTask)
|
|
126
|
+
await heartbeatTask;
|
|
127
|
+
if (stopAfterLeaseLoss)
|
|
128
|
+
await stopAfterLeaseLoss;
|
|
129
|
+
if (leaseLost)
|
|
130
|
+
throw leaseLost;
|
|
131
|
+
releaseReason = classifyAdapterResult(result);
|
|
132
|
+
return result;
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
releaseReason = classifyAdapterError(error);
|
|
136
|
+
throw error;
|
|
137
|
+
}
|
|
138
|
+
finally {
|
|
139
|
+
clearInterval(heartbeatTimer);
|
|
140
|
+
const payload = { ...context.lease, reason: releaseReason };
|
|
141
|
+
let released = false;
|
|
142
|
+
let lastError;
|
|
143
|
+
for (let attempt = 0; attempt < 2 && !released; attempt++) {
|
|
144
|
+
try {
|
|
145
|
+
await release(payload);
|
|
146
|
+
released = true;
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
lastError = error;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (!released) {
|
|
153
|
+
warn(`Adapter lease release acknowledgement failed for ${payload.requestId}: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
function classifyAdapterError(error) {
|
|
158
|
+
const code = error && typeof error === 'object' && typeof error.code === 'string'
|
|
159
|
+
? error.code
|
|
160
|
+
: '';
|
|
161
|
+
if (code === 'AUTH_REQUIRED' || /CAPTCHA|MFA|VERIFICATION/.test(code))
|
|
162
|
+
return 'auth_gate';
|
|
163
|
+
if (code === 'RATE_LIMITED')
|
|
164
|
+
return 'rate_limited';
|
|
165
|
+
if (code === 'TIMEOUT')
|
|
166
|
+
return 'timeout';
|
|
167
|
+
return 'error';
|
|
168
|
+
}
|
|
169
|
+
function classifyAdapterResult(result) {
|
|
170
|
+
const rows = Array.isArray(result) ? result : [result];
|
|
171
|
+
const statuses = rows
|
|
172
|
+
.filter(row => row && typeof row === 'object')
|
|
173
|
+
.map(row => String(row.status ?? '').toLowerCase());
|
|
174
|
+
if (statuses.some(status => /auth|login|captcha|verification|mfa/.test(status)))
|
|
175
|
+
return 'auth_gate';
|
|
176
|
+
if (statuses.some(status => /rate.?limit/.test(status)))
|
|
177
|
+
return 'rate_limited';
|
|
178
|
+
if (statuses.some(status => status === 'partial'))
|
|
179
|
+
return 'partial';
|
|
180
|
+
if (statuses.length > 0 && statuses.every(status => status.startsWith('failed') || status.startsWith('failure')))
|
|
181
|
+
return 'failed';
|
|
182
|
+
return 'success';
|
|
183
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { AdapterLease } from './adapter-scheduler.js';
|
|
2
|
+
export interface AdapterExecutionContext {
|
|
3
|
+
lease: AdapterLease;
|
|
4
|
+
}
|
|
5
|
+
export declare function getAdapterExecutionContext(): AdapterExecutionContext | undefined;
|
|
6
|
+
export declare function runWithAdapterExecutionContext<T>(context: AdapterExecutionContext, operation: () => Promise<T>): Promise<T>;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
+
const storage = new AsyncLocalStorage();
|
|
3
|
+
export function getAdapterExecutionContext() {
|
|
4
|
+
return storage.getStore();
|
|
5
|
+
}
|
|
6
|
+
export function runWithAdapterExecutionContext(context, operation) {
|
|
7
|
+
return storage.run(context, operation);
|
|
8
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
export type AdapterPoolCloseReason = 'auth_gate' | 'rate_limited';
|
|
2
|
+
export type AdapterReleaseReason = 'success' | 'partial' | 'failed' | AdapterPoolCloseReason | 'timeout' | 'error' | 'cancelled';
|
|
3
|
+
export interface AdapterLeaseRequest {
|
|
4
|
+
requestId: string;
|
|
5
|
+
contextId: string;
|
|
6
|
+
surface: 'adapter';
|
|
7
|
+
site: string;
|
|
8
|
+
adapterSession: string;
|
|
9
|
+
sessionKey: string;
|
|
10
|
+
queueTimeoutMs: number;
|
|
11
|
+
maxParallel: number;
|
|
12
|
+
}
|
|
13
|
+
export interface AdapterLease {
|
|
14
|
+
leaseId: string;
|
|
15
|
+
requestId: string;
|
|
16
|
+
poolKey: string;
|
|
17
|
+
contextId: string;
|
|
18
|
+
surface: 'adapter';
|
|
19
|
+
site: string;
|
|
20
|
+
adapterSession: string;
|
|
21
|
+
sessionKey: string;
|
|
22
|
+
generation: number;
|
|
23
|
+
grantedAt: number;
|
|
24
|
+
heartbeatDeadline: number;
|
|
25
|
+
}
|
|
26
|
+
export interface AdapterLeaseRelease extends AdapterLease {
|
|
27
|
+
reason: AdapterReleaseReason;
|
|
28
|
+
}
|
|
29
|
+
export declare class AdapterSchedulerError extends Error {
|
|
30
|
+
readonly code: string;
|
|
31
|
+
constructor(code: string, message: string);
|
|
32
|
+
}
|
|
33
|
+
export interface AdapterResourceGrant {
|
|
34
|
+
grantId: string;
|
|
35
|
+
leaseId: string;
|
|
36
|
+
keys: string[];
|
|
37
|
+
}
|
|
38
|
+
export interface AdapterSchedulerOptions {
|
|
39
|
+
now?: () => number;
|
|
40
|
+
leaseExpiryMs?: number;
|
|
41
|
+
runtimeCeiling?: number;
|
|
42
|
+
releasedLeaseRetentionMs?: number;
|
|
43
|
+
}
|
|
44
|
+
export declare class AdapterScheduler {
|
|
45
|
+
private readonly now;
|
|
46
|
+
private readonly leaseExpiryMs;
|
|
47
|
+
private readonly runtimeCeiling;
|
|
48
|
+
private readonly releasedLeaseRetentionMs;
|
|
49
|
+
private readonly pools;
|
|
50
|
+
private readonly releasedLeaseIds;
|
|
51
|
+
private readonly generations;
|
|
52
|
+
private readonly resourceOwners;
|
|
53
|
+
private readonly resourceGrants;
|
|
54
|
+
private readonly resourceQueue;
|
|
55
|
+
private sequence;
|
|
56
|
+
constructor(options?: AdapterSchedulerOptions);
|
|
57
|
+
acquire(request: AdapterLeaseRequest): Promise<AdapterLease>;
|
|
58
|
+
heartbeat(identity: AdapterLease): AdapterLease;
|
|
59
|
+
assertLease(identity: AdapterLease): AdapterLease;
|
|
60
|
+
release(release: AdapterLeaseRelease): boolean;
|
|
61
|
+
acquireResources(leaseIdentity: AdapterLease, rawKeys: string[], timeoutMs: number): Promise<AdapterResourceGrant>;
|
|
62
|
+
releaseResources(leaseIdentity: AdapterLease, grantId: string): boolean;
|
|
63
|
+
cancel(requestId: string, code?: string): boolean;
|
|
64
|
+
sweepExpired(): void;
|
|
65
|
+
reset(): void;
|
|
66
|
+
snapshot(): {
|
|
67
|
+
running: number;
|
|
68
|
+
queued: number;
|
|
69
|
+
pools: number;
|
|
70
|
+
};
|
|
71
|
+
resourceSnapshot(): {
|
|
72
|
+
locked: number;
|
|
73
|
+
queued: number;
|
|
74
|
+
grants: number;
|
|
75
|
+
};
|
|
76
|
+
private schedule;
|
|
77
|
+
private scheduleResources;
|
|
78
|
+
private releaseAllResourcesForLease;
|
|
79
|
+
private pruneReleasedLeaseIds;
|
|
80
|
+
private rejectResourceWaitersForLease;
|
|
81
|
+
private requireLease;
|
|
82
|
+
private getOrCreatePool;
|
|
83
|
+
private removeDrainedPool;
|
|
84
|
+
private poolClosedError;
|
|
85
|
+
private validateRequest;
|
|
86
|
+
}
|