dsh-plugin-lcsc 0.3.4 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +184 -181
- package/cn.js +808 -808
- package/cordis.patch.yml +28 -28
- package/data.js +1816 -1816
- package/icon.svg +9 -9
- package/index.js +2459 -2207
- package/locale/en.json +6 -6
- package/locale/zh.json +6 -6
- package/package.json +52 -52
package/cn.js
CHANGED
|
@@ -1,808 +1,808 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* LCSC domestic (立创商城) CNY price/stock adapter.
|
|
3
|
-
*
|
|
4
|
-
* The domestic store sits behind an Aliyun WAF that challenges plain HTTP
|
|
5
|
-
* requests, but a real browser executes the challenge and then serves the
|
|
6
|
-
* product page with `<script id="__NEXT_DATA__">` containing the official CNY
|
|
7
|
-
* price and domestic stock. This module renders `item.szlcsc.com/<productId>.html`
|
|
8
|
-
* with the locally installed Chrome (`--headless=new --dump-dom`) and caches the
|
|
9
|
-
* result on disk for 24 h.
|
|
10
|
-
*
|
|
11
|
-
* Flow: LCSC code -> EasyEDA / lceda.cn product id -> Chrome render -> parse
|
|
12
|
-
* __NEXT_DATA__ -> cache. Every failure is returned as `{ ok: false, reason }`
|
|
13
|
-
* so callers can fall back to the USD estimate instead of failing the batch.
|
|
14
|
-
*
|
|
15
|
-
* Speed: a single persistent Chrome instance is kept alive and driven over the
|
|
16
|
-
* DevTools protocol (`--remote-debugging-pipe`), opening one tab per product.
|
|
17
|
-
* Cold start/warm-up costs ~4-5 s (WAF challenge), after which each product is
|
|
18
|
-
* ~1-2 s; 4-8 tabs in parallel warm a 28-part BOM in well under a minute.
|
|
19
|
-
* Chrome is started on demand, reused across calls, and auto-closed after an
|
|
20
|
-
* idle TTL. If the pipe/CDP path is unavailable, it falls back to the legacy
|
|
21
|
-
* per-page `--dump-dom` spawn (slower but dependency-free).
|
|
22
|
-
*/
|
|
23
|
-
|
|
24
|
-
import { spawn } from 'node:child_process'
|
|
25
|
-
import fs from 'node:fs'
|
|
26
|
-
import os from 'node:os'
|
|
27
|
-
import path from 'node:path'
|
|
28
|
-
|
|
29
|
-
const EASYEDA_URL = 'https://easyeda.com/api/products/'
|
|
30
|
-
const LCEDA_URL = 'https://lceda.cn/api/products/'
|
|
31
|
-
const ITEM_URL = 'https://item.szlcsc.com/'
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
* Common Chrome/Chromium locations. Windows roots are read from the
|
|
35
|
-
* environment so the list follows the real install (drive/ProgramFiles can be
|
|
36
|
-
* relocated) instead of a hard-coded path.
|
|
37
|
-
* @returns {string[]}
|
|
38
|
-
*/
|
|
39
|
-
function chromeCandidates() {
|
|
40
|
-
const candidates = [
|
|
41
|
-
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
|
42
|
-
'/Applications/Chromium.app/Contents/MacOS/Chromium',
|
|
43
|
-
'/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
|
|
44
|
-
'/usr/bin/google-chrome',
|
|
45
|
-
'/usr/bin/google-chrome-stable',
|
|
46
|
-
'/usr/bin/chromium',
|
|
47
|
-
'/usr/bin/chromium-browser',
|
|
48
|
-
'/snap/bin/chromium',
|
|
49
|
-
]
|
|
50
|
-
if (process.platform === 'win32') {
|
|
51
|
-
const roots = [process.env.ProgramFiles, process.env['ProgramFiles(x86)'], process.env.ProgramW6432, process.env.LOCALAPPDATA]
|
|
52
|
-
.filter((root) => typeof root === 'string' && root !== '')
|
|
53
|
-
const relatives = [
|
|
54
|
-
['Google', 'Chrome', 'Application', 'chrome.exe'],
|
|
55
|
-
['Chromium', 'Application', 'chrome.exe'],
|
|
56
|
-
['Microsoft', 'Edge', 'Application', 'msedge.exe'],
|
|
57
|
-
]
|
|
58
|
-
for (const root of roots) {
|
|
59
|
-
for (const relative of relatives) candidates.push(path.join(root, ...relative))
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
return candidates
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
const DEFAULT_USER_AGENT =
|
|
66
|
-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' +
|
|
67
|
-
'(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
|
|
68
|
-
|
|
69
|
-
/** Ali yun WAF challenge markers: page is not the product page we asked for. */
|
|
70
|
-
const WAF_HINT_RE = /aliyun|acw_sc__v2|_____tmd_____|验证码|captcha|challenge-platform|error code.*?5\d\d/i
|
|
71
|
-
|
|
72
|
-
/**
|
|
73
|
-
* True when the rendered HTML is a WAF challenge page rather than a product
|
|
74
|
-
* page. A page carrying `__NEXT_DATA__` is always the real one.
|
|
75
|
-
* @param {unknown} html
|
|
76
|
-
*/
|
|
77
|
-
export function looksLikeWafChallenge(html) {
|
|
78
|
-
if (typeof html !== 'string' || html === '') return false
|
|
79
|
-
if (html.includes('__NEXT_DATA__')) return false
|
|
80
|
-
return WAF_HINT_RE.test(html)
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
/** One retry for transient failures; the domestic site challenges in bursts. */
|
|
84
|
-
const RETRY_DELAY_MS = 1500
|
|
85
|
-
const RETRYABLE_FAILURES = new Set(['szlcsc-waf-challenge', 'chrome-render-empty', 'page-parse-failed'])
|
|
86
|
-
|
|
87
|
-
/**
|
|
88
|
-
* How to fix a missing Chrome, phrased so an agent can act on it directly.
|
|
89
|
-
* @param {{ path?: string, source?: string, tried: string[], configPath?: string, envPath?: string }} info
|
|
90
|
-
* @returns {string}
|
|
91
|
-
*/
|
|
92
|
-
export function chromeHint(info) {
|
|
93
|
-
const parts = ['未找到可用的 Chrome/Chromium,国内站人民币价与国内库存不可用(已回退为 USD 估算)。']
|
|
94
|
-
if (info.configPath !== undefined || info.envPath !== undefined) {
|
|
95
|
-
const missing = []
|
|
96
|
-
if (info.configPath !== undefined) missing.push(`config.chromePath=${info.configPath}`)
|
|
97
|
-
if (info.envPath !== undefined) missing.push(`CHROME_BIN=${info.envPath}`)
|
|
98
|
-
parts.push(`当前指定的路径不存在:${missing.join(',')}。`)
|
|
99
|
-
}
|
|
100
|
-
parts.push(`已尝试 ${info.tried.length} 个常见路径${info.tried.length === 0 ? '' : `(${info.tried.slice(0, 5).join(' ; ')}${info.tried.length > 5 ? ' …' : ''})`}。`)
|
|
101
|
-
parts.push(
|
|
102
|
-
'请任选其一修复后重试:① 在本插件配置里加 chromePath: "<本机 Chrome 可执行文件绝对路径>"' +
|
|
103
|
-
'(profile 的 cordis.patch.yml,条目 id 为 lcsc-parts);② 给宿主进程设置环境变量 CHROME_BIN。' +
|
|
104
|
-
'定位方法:Windows `where chrome` 或查看 %ProgramFiles%\\Google\\Chrome\\Application\\chrome.exe;' +
|
|
105
|
-
'macOS `ls /Applications | grep -i chrome`;Linux `which google-chrome chromium`。',
|
|
106
|
-
)
|
|
107
|
-
return parts.join('')
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
/**
|
|
111
|
-
* Locate a Chrome/Chromium binary and explain where it came from.
|
|
112
|
-
* Precedence: explicit config → CHROME_BIN → auto-discovery.
|
|
113
|
-
* @param {string} [explicit]
|
|
114
|
-
* @returns {{ path?: string, source?: 'config' | 'chrome_bin' | 'auto', tried: string[], configPath?: string, envPath?: string }}
|
|
115
|
-
*/
|
|
116
|
-
export function findChromeInfo(explicit) {
|
|
117
|
-
const configPath = typeof explicit === 'string' && explicit !== '' ? explicit : undefined
|
|
118
|
-
if (configPath !== undefined && fs.existsSync(configPath)) return { path: configPath, source: 'config', tried: [], configPath }
|
|
119
|
-
const envPath = typeof process.env.CHROME_BIN === 'string' && process.env.CHROME_BIN !== '' ? process.env.CHROME_BIN : undefined
|
|
120
|
-
if (envPath !== undefined && fs.existsSync(envPath)) return { path: envPath, source: 'chrome_bin', tried: [], envPath }
|
|
121
|
-
const tried = []
|
|
122
|
-
for (const candidate of chromeCandidates()) {
|
|
123
|
-
if (tried.includes(candidate)) continue
|
|
124
|
-
tried.push(candidate)
|
|
125
|
-
if (fs.existsSync(candidate)) return { path: candidate, source: 'auto', tried, configPath, envPath }
|
|
126
|
-
}
|
|
127
|
-
return { path: undefined, source: undefined, tried, configPath, envPath }
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
/** Locate a Chrome/Chromium binary (compatibility wrapper around findChromeInfo). */
|
|
131
|
-
export function findChrome(explicit) {
|
|
132
|
-
return findChromeInfo(explicit).path
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
/** Extract the `__NEXT_DATA__` JSON from an HTML document (or undefined). */
|
|
136
|
-
export function extractNextData(html) {
|
|
137
|
-
const match = String(html ?? '').match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/)
|
|
138
|
-
if (match === null) return undefined
|
|
139
|
-
try {
|
|
140
|
-
return JSON.parse(match[1])
|
|
141
|
-
} catch {
|
|
142
|
-
return undefined
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
/** Chrome profile subdirectories that are pure cache/garbage and can be deleted freely. */
|
|
147
|
-
const CHROME_PROFILE_JUNK = [
|
|
148
|
-
'Default/Cache',
|
|
149
|
-
'Default/Code Cache',
|
|
150
|
-
'Default/GPUCache',
|
|
151
|
-
'Default/DawnCache',
|
|
152
|
-
'Default/DawnGraphiteCache',
|
|
153
|
-
'Default/DawnWebGPUCache',
|
|
154
|
-
'Default/GraphiteDawnCache',
|
|
155
|
-
'Default/ShaderCache',
|
|
156
|
-
'Default/Service Worker/CacheStorage',
|
|
157
|
-
'Default/Service Worker/ScriptCache',
|
|
158
|
-
'Default/Media Cache',
|
|
159
|
-
'Default/Session Storage',
|
|
160
|
-
'Default/WebStorage',
|
|
161
|
-
'Crashpad',
|
|
162
|
-
'ShaderCache',
|
|
163
|
-
'GrShaderCache',
|
|
164
|
-
'BrowserMetrics',
|
|
165
|
-
'component_crx_cache',
|
|
166
|
-
'extensions_crx_cache',
|
|
167
|
-
]
|
|
168
|
-
|
|
169
|
-
/**
|
|
170
|
-
* Delete cache-only subdirectories from a Chrome profile, keeping cookies and
|
|
171
|
-
* preferences (the WAF challenge cookie lives in the profile).
|
|
172
|
-
* @param {string} profileDir
|
|
173
|
-
*/
|
|
174
|
-
export function pruneChromeProfile(profileDir) {
|
|
175
|
-
for (const relative of CHROME_PROFILE_JUNK) {
|
|
176
|
-
try {
|
|
177
|
-
fs.rmSync(path.join(profileDir, relative), { recursive: true, force: true })
|
|
178
|
-
} catch {
|
|
179
|
-
// best effort
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
/**
|
|
185
|
-
* Prune the JSON result cache: drop entries older than `ttlMs`, then keep at
|
|
186
|
-
* most `maxEntries` newest entries. This keeps `~/.cache/dsh-plugin-lcsc/cn`
|
|
187
|
-
* bounded instead of accumulating one file per part ever queried.
|
|
188
|
-
* @param {string} cacheDir
|
|
189
|
-
* @param {{ ttlMs?: number, maxEntries?: number }} [options]
|
|
190
|
-
* @returns {{ removed: number, kept: number }}
|
|
191
|
-
*/
|
|
192
|
-
export function pruneJsonCache(cacheDir, options = {}) {
|
|
193
|
-
const ttlMs = Number.isFinite(options.ttlMs) ? options.ttlMs : 24 * 60 * 60 * 1000
|
|
194
|
-
const maxEntries = Number.isFinite(options.maxEntries) ? Math.max(1, Math.floor(options.maxEntries)) : 800
|
|
195
|
-
let names
|
|
196
|
-
try {
|
|
197
|
-
names = fs.readdirSync(cacheDir).filter((name) => name.endsWith('.json'))
|
|
198
|
-
} catch {
|
|
199
|
-
return { removed: 0, kept: 0 }
|
|
200
|
-
}
|
|
201
|
-
const now = Date.now()
|
|
202
|
-
/** @type {{ file: string, fetchedAt: number }[]} */
|
|
203
|
-
const alive = []
|
|
204
|
-
let removed = 0
|
|
205
|
-
for (const name of names) {
|
|
206
|
-
const file = path.join(cacheDir, name)
|
|
207
|
-
let record
|
|
208
|
-
try {
|
|
209
|
-
record = JSON.parse(fs.readFileSync(file, 'utf8'))
|
|
210
|
-
} catch {
|
|
211
|
-
record = undefined
|
|
212
|
-
}
|
|
213
|
-
const fetchedAt = record === undefined ? Number.NaN : Date.parse(record.fetched_at)
|
|
214
|
-
if (!Number.isFinite(fetchedAt) || now - fetchedAt > ttlMs) {
|
|
215
|
-
try {
|
|
216
|
-
fs.rmSync(file, { force: true })
|
|
217
|
-
removed += 1
|
|
218
|
-
} catch {
|
|
219
|
-
// best effort
|
|
220
|
-
}
|
|
221
|
-
continue
|
|
222
|
-
}
|
|
223
|
-
alive.push({ file, fetchedAt })
|
|
224
|
-
}
|
|
225
|
-
if (alive.length > maxEntries) {
|
|
226
|
-
alive.sort((left, right) => left.fetchedAt - right.fetchedAt)
|
|
227
|
-
for (const entry of alive.slice(0, alive.length - maxEntries)) {
|
|
228
|
-
try {
|
|
229
|
-
fs.rmSync(entry.file, { force: true })
|
|
230
|
-
removed += 1
|
|
231
|
-
} catch {
|
|
232
|
-
// best effort
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
return { removed, kept: maxEntries }
|
|
236
|
-
}
|
|
237
|
-
return { removed, kept: alive.length }
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
/**
|
|
241
|
-
* Parse a rendered `item.szlcsc.com` page into the official domestic record.
|
|
242
|
-
* @param {string} html
|
|
243
|
-
* @returns {{ price?: number, model?: string, brand?: string, package?: string, stock?: number, min_buy?: number, packing?: string, unit?: string, category?: string, desc?: string, product_id?: string } | undefined}
|
|
244
|
-
*/
|
|
245
|
-
export function parseItemHtml(html) {
|
|
246
|
-
const text = String(html ?? '')
|
|
247
|
-
const nextData = extractNextData(text)
|
|
248
|
-
if (nextData === undefined) {
|
|
249
|
-
// JSON-LD fallback for pages that omit __NEXT_DATA__.
|
|
250
|
-
const fallback = text.match(/"priceCurrency":"CNY","price":([0-9.]+)/)
|
|
251
|
-
if (fallback === null) return undefined
|
|
252
|
-
return { price: Number(fallback[1]) }
|
|
253
|
-
}
|
|
254
|
-
const pageProps = nextData?.props?.pageProps ?? {}
|
|
255
|
-
const webData = pageProps.webData ?? {}
|
|
256
|
-
const productRecord = webData.productRecord ?? {}
|
|
257
|
-
const price = Number(pageProps.price)
|
|
258
|
-
const result = {
|
|
259
|
-
price: Number.isFinite(price) ? price : undefined,
|
|
260
|
-
model: stringOrUndefined(productRecord.productModel),
|
|
261
|
-
brand: stringOrUndefined(productRecord.productGradePlateName),
|
|
262
|
-
package: stringOrUndefined(productRecord.encapsulationModel),
|
|
263
|
-
stock: integerOrUndefined(productRecord.stockNumber),
|
|
264
|
-
min_buy: integerOrUndefined(productRecord.minBuyNumber),
|
|
265
|
-
packing: stringOrUndefined(productRecord.productArrange),
|
|
266
|
-
unit: stringOrUndefined(productRecord.productUnit),
|
|
267
|
-
category: stringOrUndefined(productRecord.productType),
|
|
268
|
-
desc: stringOrUndefined(productRecord.productName) ?? stringOrUndefined(productRecord.remark),
|
|
269
|
-
product_id: productRecord.productId ?? productRecord.productCode,
|
|
270
|
-
}
|
|
271
|
-
if (result.price === undefined) {
|
|
272
|
-
const fallback = text.match(/"priceCurrency":"CNY","price":([0-9.]+)/)
|
|
273
|
-
if (fallback !== null) result.price = Number(fallback[1])
|
|
274
|
-
}
|
|
275
|
-
return result
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
/** @param {unknown} value @returns {string | undefined} */
|
|
279
|
-
function stringOrUndefined(value) {
|
|
280
|
-
if (typeof value !== 'string') return undefined
|
|
281
|
-
const text = value.trim()
|
|
282
|
-
return text === '' ? undefined : text
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
/** @param {unknown} value @returns {number | undefined} */
|
|
286
|
-
function integerOrUndefined(value) {
|
|
287
|
-
const number = typeof value === 'string' ? Number(value) : value
|
|
288
|
-
return typeof number === 'number' && Number.isInteger(number) ? number : undefined
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
/** @param {number} ms */
|
|
292
|
-
function sleep(ms) {
|
|
293
|
-
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
/**
|
|
297
|
-
* Create the domestic-CNY adapter.
|
|
298
|
-
* @param {{
|
|
299
|
-
* chromePath?: string,
|
|
300
|
-
* cacheDir?: string,
|
|
301
|
-
* cacheTtlMs?: number,
|
|
302
|
-
* timeoutMs?: number,
|
|
303
|
-
* concurrency?: number,
|
|
304
|
-
* browserIdleTtlMs?: number,
|
|
305
|
-
* verbose?: boolean,
|
|
306
|
-
* logger?: { warn?: (message: string) => void, info?: (message: string) => void },
|
|
307
|
-
* }} [options]
|
|
308
|
-
*/
|
|
309
|
-
export function createCnAdapter(options = {}) {
|
|
310
|
-
const chromeInfo = findChromeInfo(options.chromePath)
|
|
311
|
-
const chromePath = chromeInfo.path
|
|
312
|
-
const chromeHintText = chromePath === undefined ? chromeHint(chromeInfo) : undefined
|
|
313
|
-
const cacheDir =
|
|
314
|
-
typeof options.cacheDir === 'string' && options.cacheDir !== ''
|
|
315
|
-
? options.cacheDir
|
|
316
|
-
: path.join(os.homedir(), '.cache', 'dsh-plugin-lcsc', 'cn')
|
|
317
|
-
const profileDir = path.join(cacheDir, 'chrome-profile')
|
|
318
|
-
const cacheTtlMs = Number.isFinite(options.cacheTtlMs) ? options.cacheTtlMs : 24 * 60 * 60 * 1000
|
|
319
|
-
const timeoutMs = Number.isFinite(options.timeoutMs) ? options.timeoutMs : 30000
|
|
320
|
-
const concurrency = Math.max(1, Math.min(8, Math.floor(options.concurrency ?? 2)))
|
|
321
|
-
const browserIdleTtlMs = Number.isFinite(options.browserIdleTtlMs) ? options.browserIdleTtlMs : 120000
|
|
322
|
-
const cacheMaxEntries = Number.isFinite(options.cacheMaxEntries) ? Math.max(1, Math.floor(options.cacheMaxEntries)) : 800
|
|
323
|
-
const legacyProfileDir = path.join(cacheDir, 'legacy-profile')
|
|
324
|
-
let lastJsonPruneAt = 0
|
|
325
|
-
let queryCount = 0
|
|
326
|
-
const logger = options.logger
|
|
327
|
-
const memory = new Map()
|
|
328
|
-
let active = 0
|
|
329
|
-
/** @type {(() => void)[]} */
|
|
330
|
-
const waiting = []
|
|
331
|
-
|
|
332
|
-
/** Run `fn` under the adapter concurrency limit. */
|
|
333
|
-
async function withSlot(fn) {
|
|
334
|
-
while (active >= concurrency) {
|
|
335
|
-
await new Promise((resolve) => waiting.push(resolve))
|
|
336
|
-
}
|
|
337
|
-
active += 1
|
|
338
|
-
try {
|
|
339
|
-
return await fn()
|
|
340
|
-
} finally {
|
|
341
|
-
active -= 1
|
|
342
|
-
const next = waiting.shift()
|
|
343
|
-
if (next !== undefined) next()
|
|
344
|
-
}
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
/** HTTP GET JSON using the global fetch (host Node >= 20). */
|
|
348
|
-
async function getJson(url) {
|
|
349
|
-
const response = await fetch(url, {
|
|
350
|
-
headers: { 'User-Agent': DEFAULT_USER_AGENT, Accept: 'application/json' },
|
|
351
|
-
signal: AbortSignal.timeout(timeoutMs),
|
|
352
|
-
})
|
|
353
|
-
if (!response.ok) throw new Error(`HTTP ${response.status} for ${url}`)
|
|
354
|
-
return response.json()
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
/* ------------------------- persistent Chrome (CDP) ----------------------- */
|
|
358
|
-
/**
|
|
359
|
-
* @typedef {{
|
|
360
|
-
* child: import('node:child_process').ChildProcess,
|
|
361
|
-
* write: NodeJS.WritableStream,
|
|
362
|
-
* read: NodeJS.ReadableStream,
|
|
363
|
-
* pending: Map<number, { resolve: (value: any) => void, reject: (reason: Error) => void, timer: NodeJS.Timeout }>,
|
|
364
|
-
* nextId: number,
|
|
365
|
-
* buffer: string,
|
|
366
|
-
* closed: boolean,
|
|
367
|
-
* }} BrowserState
|
|
368
|
-
*/
|
|
369
|
-
/** @type {BrowserState | undefined} */
|
|
370
|
-
let browser
|
|
371
|
-
/** @type {NodeJS.Timeout | undefined} */
|
|
372
|
-
let idleTimer
|
|
373
|
-
let cdpBroken = false
|
|
374
|
-
let cdpSessionErrors = 0
|
|
375
|
-
|
|
376
|
-
function shutdownBrowser() {
|
|
377
|
-
if (idleTimer !== undefined) {
|
|
378
|
-
clearTimeout(idleTimer)
|
|
379
|
-
idleTimer = undefined
|
|
380
|
-
}
|
|
381
|
-
const state = browser
|
|
382
|
-
browser = undefined
|
|
383
|
-
if (state === undefined) return
|
|
384
|
-
state.closed = true
|
|
385
|
-
for (const entry of state.pending.values()) {
|
|
386
|
-
clearTimeout(entry.timer)
|
|
387
|
-
entry.reject(new Error('chrome-shutdown'))
|
|
388
|
-
}
|
|
389
|
-
state.pending.clear()
|
|
390
|
-
try {
|
|
391
|
-
state.child.kill('SIGKILL')
|
|
392
|
-
} catch {
|
|
393
|
-
// already gone
|
|
394
|
-
}
|
|
395
|
-
// Keep cookies for the WAF challenge, drop HTTP/Code caches so the profile
|
|
396
|
-
// does not silently grow.
|
|
397
|
-
pruneChromeProfile(profileDir)
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
function touchIdle() {
|
|
401
|
-
if (idleTimer !== undefined) clearTimeout(idleTimer)
|
|
402
|
-
if (browserIdleTtlMs <= 0) return
|
|
403
|
-
idleTimer = setTimeout(() => shutdownBrowser(), browserIdleTtlMs)
|
|
404
|
-
idleTimer.unref?.()
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
/** Start Chrome once and resolve when its DevTools pipe answers. */
|
|
408
|
-
async function startBrowser() {
|
|
409
|
-
if (browser !== undefined && browser.closed !== true) return browser
|
|
410
|
-
if (chromePath === undefined) throw new Error('chrome-not-found')
|
|
411
|
-
fs.mkdirSync(profileDir, { recursive: true })
|
|
412
|
-
// Clean caches left behind by an interrupted previous run.
|
|
413
|
-
pruneChromeProfile(profileDir)
|
|
414
|
-
const child = spawn(
|
|
415
|
-
chromePath,
|
|
416
|
-
[
|
|
417
|
-
'--headless=new',
|
|
418
|
-
'--disable-gpu',
|
|
419
|
-
'--no-first-run',
|
|
420
|
-
'--no-default-browser-check',
|
|
421
|
-
'--disable-background-networking',
|
|
422
|
-
'--disable-sync',
|
|
423
|
-
'--mute-audio',
|
|
424
|
-
'--disk-cache-size=52428800', // hard cap: 50 MiB HTTP cache
|
|
425
|
-
'--media-cache-size=10485760', // hard cap: 10 MiB media cache
|
|
426
|
-
'--remote-debugging-pipe',
|
|
427
|
-
`--user-data-dir=${profileDir}`,
|
|
428
|
-
'about:blank',
|
|
429
|
-
],
|
|
430
|
-
{ stdio: ['ignore', 'ignore', 'ignore', 'pipe', 'pipe'] },
|
|
431
|
-
)
|
|
432
|
-
const write = /** @type {NodeJS.WritableStream} */ (child.stdio[3])
|
|
433
|
-
const read = /** @type {NodeJS.ReadableStream} */ (child.stdio[4])
|
|
434
|
-
if (write === null || read === null) {
|
|
435
|
-
try {
|
|
436
|
-
child.kill('SIGKILL')
|
|
437
|
-
} catch {
|
|
438
|
-
// ignore
|
|
439
|
-
}
|
|
440
|
-
throw new Error('chrome-pipe-unavailable')
|
|
441
|
-
}
|
|
442
|
-
/** @type {BrowserState} */
|
|
443
|
-
const state = { child, write, read, pending: new Map(), nextId: 1, buffer: '', closed: false }
|
|
444
|
-
read.on('data', (chunk) => {
|
|
445
|
-
state.buffer += chunk.toString('utf8')
|
|
446
|
-
const parts = state.buffer.split('\u0000')
|
|
447
|
-
state.buffer = parts.pop() ?? ''
|
|
448
|
-
for (const part of parts) {
|
|
449
|
-
if (part.trim() === '') continue
|
|
450
|
-
let message
|
|
451
|
-
try {
|
|
452
|
-
message = JSON.parse(part)
|
|
453
|
-
} catch {
|
|
454
|
-
continue
|
|
455
|
-
}
|
|
456
|
-
if (message.id === undefined) continue
|
|
457
|
-
const entry = state.pending.get(message.id)
|
|
458
|
-
if (entry === undefined) continue
|
|
459
|
-
state.pending.delete(message.id)
|
|
460
|
-
clearTimeout(entry.timer)
|
|
461
|
-
if (message.error !== undefined) entry.reject(new Error(JSON.stringify(message.error)))
|
|
462
|
-
else entry.resolve(message.result ?? message)
|
|
463
|
-
}
|
|
464
|
-
})
|
|
465
|
-
child.on('error', (error) => {
|
|
466
|
-
state.closed = true
|
|
467
|
-
if (browser === state) browser = undefined
|
|
468
|
-
for (const entry of state.pending.values()) {
|
|
469
|
-
clearTimeout(entry.timer)
|
|
470
|
-
entry.reject(error instanceof Error ? error : new Error(String(error)))
|
|
471
|
-
}
|
|
472
|
-
state.pending.clear()
|
|
473
|
-
})
|
|
474
|
-
child.on('exit', () => {
|
|
475
|
-
state.closed = true
|
|
476
|
-
if (browser === state) browser = undefined
|
|
477
|
-
for (const entry of state.pending.values()) {
|
|
478
|
-
clearTimeout(entry.timer)
|
|
479
|
-
entry.reject(new Error('chrome-exited'))
|
|
480
|
-
}
|
|
481
|
-
state.pending.clear()
|
|
482
|
-
})
|
|
483
|
-
browser = state
|
|
484
|
-
// handshake
|
|
485
|
-
await cdpSend(state, 'Browser.getVersion', {}, undefined, 8000)
|
|
486
|
-
if (options.verbose === true) logger?.info?.(`[lcsc-cn] chrome started (pid ${child.pid})`)
|
|
487
|
-
return state
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
/**
|
|
491
|
-
* Send one CDP command over the pipe.
|
|
492
|
-
* @param {BrowserState} state
|
|
493
|
-
* @param {string} method
|
|
494
|
-
* @param {Record<string, unknown>} [params]
|
|
495
|
-
* @param {string} [sessionId]
|
|
496
|
-
* @param {number} [timeout]
|
|
497
|
-
*/
|
|
498
|
-
function cdpSend(state, method, params = {}, sessionId, timeout = 15000) {
|
|
499
|
-
return new Promise((resolve, reject) => {
|
|
500
|
-
if (state.closed) {
|
|
501
|
-
reject(new Error('chrome-closed'))
|
|
502
|
-
return
|
|
503
|
-
}
|
|
504
|
-
const id = state.nextId
|
|
505
|
-
state.nextId += 1
|
|
506
|
-
const timer = setTimeout(() => {
|
|
507
|
-
state.pending.delete(id)
|
|
508
|
-
reject(new Error(`cdp-timeout:${method}`))
|
|
509
|
-
}, timeout)
|
|
510
|
-
timer.unref?.()
|
|
511
|
-
state.pending.set(id, { resolve, reject, timer })
|
|
512
|
-
const payload = { id, method, params }
|
|
513
|
-
if (sessionId !== undefined) payload.sessionId = sessionId
|
|
514
|
-
state.write.write(`${JSON.stringify(payload)}\u0000`)
|
|
515
|
-
})
|
|
516
|
-
}
|
|
517
|
-
|
|
518
|
-
/** Open `url` in a new tab, wait for the rendered DOM, close the tab. */
|
|
519
|
-
async function renderViaCdp(url, timeout = timeoutMs) {
|
|
520
|
-
const state = await startBrowser()
|
|
521
|
-
const deadline = Date.now() + timeout
|
|
522
|
-
let targetId
|
|
523
|
-
let sessionId
|
|
524
|
-
try {
|
|
525
|
-
const created = await cdpSend(state, 'Target.createTarget', { url: 'about:blank' }, undefined, 10000)
|
|
526
|
-
targetId = created.targetId
|
|
527
|
-
const attached = await cdpSend(state, 'Target.attachToTarget', { targetId, flatten: true }, undefined, 10000)
|
|
528
|
-
sessionId = attached.sessionId
|
|
529
|
-
await cdpSend(state, 'Page.enable', {}, sessionId, 5000)
|
|
530
|
-
await cdpSend(state, 'Runtime.enable', {}, sessionId, 5000)
|
|
531
|
-
await cdpSend(state, 'Page.navigate', { url }, sessionId, 10000)
|
|
532
|
-
let html = ''
|
|
533
|
-
while (Date.now() < deadline) {
|
|
534
|
-
await sleep(150)
|
|
535
|
-
try {
|
|
536
|
-
const evaluated = await cdpSend(
|
|
537
|
-
state,
|
|
538
|
-
'Runtime.evaluate',
|
|
539
|
-
{ expression: 'document.documentElement.outerHTML', returnByValue: true },
|
|
540
|
-
sessionId,
|
|
541
|
-
5000,
|
|
542
|
-
)
|
|
543
|
-
html = typeof evaluated?.result?.value === 'string' ? evaluated.result.value : ''
|
|
544
|
-
} catch {
|
|
545
|
-
// navigation might still be in flight; retry until the deadline
|
|
546
|
-
}
|
|
547
|
-
if (html.includes('__NEXT_DATA__') && /"price"\s*:/.test(html)) return html
|
|
548
|
-
}
|
|
549
|
-
return html
|
|
550
|
-
} finally {
|
|
551
|
-
if (sessionId !== undefined && state.closed !== true) {
|
|
552
|
-
try {
|
|
553
|
-
await cdpSend(state, 'Target.detachFromTarget', { sessionId }, undefined, 3000)
|
|
554
|
-
} catch {
|
|
555
|
-
// ignore
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
if (targetId !== undefined && state.closed !== true) {
|
|
559
|
-
try {
|
|
560
|
-
await cdpSend(state, 'Target.closeTarget', { targetId }, undefined, 3000)
|
|
561
|
-
} catch {
|
|
562
|
-
// ignore
|
|
563
|
-
}
|
|
564
|
-
}
|
|
565
|
-
touchIdle()
|
|
566
|
-
}
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
/* ------------------- legacy per-page renderer (fallback) ----------------- */
|
|
570
|
-
/** @param {string} url @param {number} timeout @returns {Promise<string>} */
|
|
571
|
-
function renderViaDumpDom(url, timeout = timeoutMs) {
|
|
572
|
-
if (chromePath === undefined) return Promise.resolve('')
|
|
573
|
-
return new Promise((resolve) => {
|
|
574
|
-
fs.mkdirSync(legacyProfileDir, { recursive: true })
|
|
575
|
-
pruneChromeProfile(legacyProfileDir)
|
|
576
|
-
const args = [
|
|
577
|
-
'--headless=new',
|
|
578
|
-
'--disable-gpu',
|
|
579
|
-
'--no-first-run',
|
|
580
|
-
'--no-default-browser-check',
|
|
581
|
-
'--disable-background-networking',
|
|
582
|
-
'--disable-sync',
|
|
583
|
-
'--mute-audio',
|
|
584
|
-
'--disk-cache-size=52428800',
|
|
585
|
-
'--media-cache-size=10485760',
|
|
586
|
-
`--user-data-dir=${legacyProfileDir}`,
|
|
587
|
-
'--dump-dom',
|
|
588
|
-
'--virtual-time-budget=6000',
|
|
589
|
-
url,
|
|
590
|
-
]
|
|
591
|
-
let finished = false
|
|
592
|
-
let stdout = ''
|
|
593
|
-
const child = spawn(chromePath, args, { stdio: ['ignore', 'pipe', 'ignore'] })
|
|
594
|
-
const timer = setTimeout(() => {
|
|
595
|
-
if (finished) return
|
|
596
|
-
finished = true
|
|
597
|
-
try {
|
|
598
|
-
child.kill('SIGKILL')
|
|
599
|
-
} catch {
|
|
600
|
-
// ignore
|
|
601
|
-
}
|
|
602
|
-
resolve(stdout)
|
|
603
|
-
}, timeout)
|
|
604
|
-
child.stdout?.on('data', (chunk) => {
|
|
605
|
-
stdout += chunk.toString()
|
|
606
|
-
})
|
|
607
|
-
child.on('error', () => {
|
|
608
|
-
if (finished) return
|
|
609
|
-
finished = true
|
|
610
|
-
clearTimeout(timer)
|
|
611
|
-
resolve('')
|
|
612
|
-
})
|
|
613
|
-
child.on('close', () => {
|
|
614
|
-
if (finished) return
|
|
615
|
-
finished = true
|
|
616
|
-
clearTimeout(timer)
|
|
617
|
-
pruneChromeProfile(legacyProfileDir)
|
|
618
|
-
resolve(stdout)
|
|
619
|
-
})
|
|
620
|
-
})
|
|
621
|
-
}
|
|
622
|
-
|
|
623
|
-
/**
|
|
624
|
-
* Render a URL to DOM: persistent CDP browser first, dump-dom fallback.
|
|
625
|
-
* @param {string} url
|
|
626
|
-
* @param {number} [timeout]
|
|
627
|
-
* @returns {Promise<string>}
|
|
628
|
-
*/
|
|
629
|
-
async function renderHtml(url, timeout = timeoutMs) {
|
|
630
|
-
if (chromePath === undefined) return ''
|
|
631
|
-
if (!cdpBroken && cdpSessionErrors < 3) {
|
|
632
|
-
try {
|
|
633
|
-
return await renderViaCdp(url, timeout)
|
|
634
|
-
} catch (error) {
|
|
635
|
-
cdpSessionErrors += 1
|
|
636
|
-
if (options.verbose === true) logger?.warn?.(`[lcsc-cn] cdp render failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
637
|
-
shutdownBrowser()
|
|
638
|
-
if (cdpSessionErrors >= 3) cdpBroken = true
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
return renderViaDumpDom(url, timeout)
|
|
642
|
-
}
|
|
643
|
-
|
|
644
|
-
/**
|
|
645
|
-
* Resolve the domestic product id for an LCSC code.
|
|
646
|
-
* @param {string} code
|
|
647
|
-
* @returns {Promise<string | undefined>}
|
|
648
|
-
*/
|
|
649
|
-
async function resolvePid(code) {
|
|
650
|
-
for (const base of [EASYEDA_URL, LCEDA_URL]) {
|
|
651
|
-
try {
|
|
652
|
-
const payload = await getJson(`${base}${encodeURIComponent(code)}/components`)
|
|
653
|
-
const result = payload?.result ?? {}
|
|
654
|
-
const pid = result?.szlcsc?.id ?? result?.lcsc?.id
|
|
655
|
-
if (pid !== undefined && pid !== null && String(pid) !== '') return String(pid)
|
|
656
|
-
} catch {
|
|
657
|
-
// try the next API
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
return undefined
|
|
661
|
-
}
|
|
662
|
-
|
|
663
|
-
/** @param {string} code */
|
|
664
|
-
function cachePath(code) {
|
|
665
|
-
return path.join(cacheDir, `${code}.json`)
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
/** @param {string} code */
|
|
669
|
-
function readCache(code) {
|
|
670
|
-
const inMemory = memory.get(code)
|
|
671
|
-
if (inMemory !== undefined && Date.now() - inMemory.ts < cacheTtlMs) {
|
|
672
|
-
return { ...inMemory.value, cached: true, cache_age_s: Math.round((Date.now() - inMemory.ts) / 1000) }
|
|
673
|
-
}
|
|
674
|
-
try {
|
|
675
|
-
const record = JSON.parse(fs.readFileSync(cachePath(code), 'utf8'))
|
|
676
|
-
const fetchedAt = Date.parse(record.fetched_at)
|
|
677
|
-
if (Number.isFinite(fetchedAt) && Date.now() - fetchedAt < cacheTtlMs) {
|
|
678
|
-
return { ...record, cached: true, cache_age_s: Math.round((Date.now() - fetchedAt) / 1000) }
|
|
679
|
-
}
|
|
680
|
-
} catch {
|
|
681
|
-
// cache miss
|
|
682
|
-
}
|
|
683
|
-
return undefined
|
|
684
|
-
}
|
|
685
|
-
|
|
686
|
-
/** @param {string} code @param {Record<string, unknown>} record */
|
|
687
|
-
function writeCache(code, record) {
|
|
688
|
-
try {
|
|
689
|
-
fs.mkdirSync(cacheDir, { recursive: true })
|
|
690
|
-
fs.writeFileSync(cachePath(code), JSON.stringify(record, null, 1), 'utf8')
|
|
691
|
-
} catch {
|
|
692
|
-
// cache write is best effort
|
|
693
|
-
}
|
|
694
|
-
memory.set(code, { ts: Date.now(), value: record })
|
|
695
|
-
}
|
|
696
|
-
|
|
697
|
-
/**
|
|
698
|
-
* Query one LCSC code. Never throws; returns `{ok:false, reason}` on failure.
|
|
699
|
-
* @param {string} code
|
|
700
|
-
* @param {{ refresh?: boolean }} [callOptions]
|
|
701
|
-
*/
|
|
702
|
-
async function query(code, callOptions = {}) {
|
|
703
|
-
const normalized = String(code ?? '').trim().toUpperCase()
|
|
704
|
-
if (normalized === '') return { ok: false, reason: 'empty-code' }
|
|
705
|
-
if (chromePath === undefined) {
|
|
706
|
-
return { ok: false, reason: 'chrome-not-found', tried: chromeInfo.tried, hint: chromeHintText }
|
|
707
|
-
}
|
|
708
|
-
queryCount += 1
|
|
709
|
-
// Prune expired JSON cache entries lazily (about once per hour / 50 hits).
|
|
710
|
-
if (queryCount === 1 || Date.now() - lastJsonPruneAt > 60 * 60 * 1000 || queryCount % 50 === 0) {
|
|
711
|
-
lastJsonPruneAt = Date.now()
|
|
712
|
-
pruneJsonCache(cacheDir, { ttlMs: cacheTtlMs, maxEntries: cacheMaxEntries })
|
|
713
|
-
}
|
|
714
|
-
const cached = callOptions.refresh === true ? undefined : readCache(normalized)
|
|
715
|
-
if (cached !== undefined && cached.ok !== false) {
|
|
716
|
-
touchIdle()
|
|
717
|
-
return cached
|
|
718
|
-
}
|
|
719
|
-
return withSlot(async () => {
|
|
720
|
-
/** @type {{ ok: false, reason: string, attempts: number } | undefined} */
|
|
721
|
-
let failure
|
|
722
|
-
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
723
|
-
if (attempt > 1) await sleep(RETRY_DELAY_MS)
|
|
724
|
-
try {
|
|
725
|
-
const pid = await resolvePid(normalized)
|
|
726
|
-
if (pid === undefined) return { ok: false, reason: 'product-id-not-found', attempts: attempt }
|
|
727
|
-
const html = await renderHtml(`${ITEM_URL}${pid}.html`)
|
|
728
|
-
if (html === '') {
|
|
729
|
-
failure = { ok: false, reason: 'chrome-render-empty', attempts: attempt }
|
|
730
|
-
continue
|
|
731
|
-
}
|
|
732
|
-
if (looksLikeWafChallenge(html)) {
|
|
733
|
-
failure = { ok: false, reason: 'szlcsc-waf-challenge', attempts: attempt }
|
|
734
|
-
continue
|
|
735
|
-
}
|
|
736
|
-
const parsed = parseItemHtml(html)
|
|
737
|
-
if (parsed === undefined || parsed.price === undefined) {
|
|
738
|
-
failure = { ok: false, reason: 'page-parse-failed', attempts: attempt }
|
|
739
|
-
continue
|
|
740
|
-
}
|
|
741
|
-
const record = {
|
|
742
|
-
ok: true,
|
|
743
|
-
lcsc: normalized,
|
|
744
|
-
product_id: pid,
|
|
745
|
-
unit: parsed.price,
|
|
746
|
-
currency: 'CNY',
|
|
747
|
-
stock: parsed.stock,
|
|
748
|
-
model: parsed.model,
|
|
749
|
-
brand: parsed.brand,
|
|
750
|
-
package: parsed.package,
|
|
751
|
-
min_buy: parsed.min_buy,
|
|
752
|
-
packing: parsed.packing,
|
|
753
|
-
unit_name: parsed.unit,
|
|
754
|
-
category: parsed.category,
|
|
755
|
-
desc: parsed.desc,
|
|
756
|
-
source: 'lcsc_cn_render',
|
|
757
|
-
fetched_at: new Date().toISOString(),
|
|
758
|
-
cache_age_s: 0,
|
|
759
|
-
cached: false,
|
|
760
|
-
}
|
|
761
|
-
writeCache(normalized, record)
|
|
762
|
-
return record
|
|
763
|
-
} catch (error) {
|
|
764
|
-
const reason = error instanceof Error ? error.message : String(error)
|
|
765
|
-
failure = { ok: false, reason, attempts: attempt }
|
|
766
|
-
if (!RETRYABLE_FAILURES.has(reason)) return failure
|
|
767
|
-
}
|
|
768
|
-
}
|
|
769
|
-
return failure ?? { ok: false, reason: 'cn-query-failed', attempts: 2 }
|
|
770
|
-
})
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
return {
|
|
774
|
-
chromePath,
|
|
775
|
-
/** Where the binary came from and what was tried; used by the tools' hints. */
|
|
776
|
-
chrome: { ...chromeInfo, hint: chromeHintText },
|
|
777
|
-
available: chromePath !== undefined,
|
|
778
|
-
cacheDir,
|
|
779
|
-
query,
|
|
780
|
-
resolvePid,
|
|
781
|
-
renderHtml,
|
|
782
|
-
clearMemory() {
|
|
783
|
-
memory.clear()
|
|
784
|
-
},
|
|
785
|
-
/** Remove this adapter's on-disk caches (JSON results + Chrome profiles). */
|
|
786
|
-
clearCache() {
|
|
787
|
-
shutdownBrowser()
|
|
788
|
-
try {
|
|
789
|
-
fs.rmSync(cacheDir, { recursive: true, force: true })
|
|
790
|
-
} catch {
|
|
791
|
-
// best effort
|
|
792
|
-
}
|
|
793
|
-
memory.clear()
|
|
794
|
-
},
|
|
795
|
-
pruneCaches() {
|
|
796
|
-
const json = pruneJsonCache(cacheDir, { ttlMs: cacheTtlMs, maxEntries: cacheMaxEntries })
|
|
797
|
-
pruneChromeProfile(profileDir)
|
|
798
|
-
pruneChromeProfile(legacyProfileDir)
|
|
799
|
-
return json
|
|
800
|
-
},
|
|
801
|
-
dispose() {
|
|
802
|
-
shutdownBrowser()
|
|
803
|
-
pruneChromeProfile(profileDir)
|
|
804
|
-
pruneChromeProfile(legacyProfileDir)
|
|
805
|
-
pruneJsonCache(cacheDir, { ttlMs: cacheTtlMs, maxEntries: cacheMaxEntries })
|
|
806
|
-
},
|
|
807
|
-
}
|
|
808
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* LCSC domestic (立创商城) CNY price/stock adapter.
|
|
3
|
+
*
|
|
4
|
+
* The domestic store sits behind an Aliyun WAF that challenges plain HTTP
|
|
5
|
+
* requests, but a real browser executes the challenge and then serves the
|
|
6
|
+
* product page with `<script id="__NEXT_DATA__">` containing the official CNY
|
|
7
|
+
* price and domestic stock. This module renders `item.szlcsc.com/<productId>.html`
|
|
8
|
+
* with the locally installed Chrome (`--headless=new --dump-dom`) and caches the
|
|
9
|
+
* result on disk for 24 h.
|
|
10
|
+
*
|
|
11
|
+
* Flow: LCSC code -> EasyEDA / lceda.cn product id -> Chrome render -> parse
|
|
12
|
+
* __NEXT_DATA__ -> cache. Every failure is returned as `{ ok: false, reason }`
|
|
13
|
+
* so callers can fall back to the USD estimate instead of failing the batch.
|
|
14
|
+
*
|
|
15
|
+
* Speed: a single persistent Chrome instance is kept alive and driven over the
|
|
16
|
+
* DevTools protocol (`--remote-debugging-pipe`), opening one tab per product.
|
|
17
|
+
* Cold start/warm-up costs ~4-5 s (WAF challenge), after which each product is
|
|
18
|
+
* ~1-2 s; 4-8 tabs in parallel warm a 28-part BOM in well under a minute.
|
|
19
|
+
* Chrome is started on demand, reused across calls, and auto-closed after an
|
|
20
|
+
* idle TTL. If the pipe/CDP path is unavailable, it falls back to the legacy
|
|
21
|
+
* per-page `--dump-dom` spawn (slower but dependency-free).
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { spawn } from 'node:child_process'
|
|
25
|
+
import fs from 'node:fs'
|
|
26
|
+
import os from 'node:os'
|
|
27
|
+
import path from 'node:path'
|
|
28
|
+
|
|
29
|
+
const EASYEDA_URL = 'https://easyeda.com/api/products/'
|
|
30
|
+
const LCEDA_URL = 'https://lceda.cn/api/products/'
|
|
31
|
+
const ITEM_URL = 'https://item.szlcsc.com/'
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Common Chrome/Chromium locations. Windows roots are read from the
|
|
35
|
+
* environment so the list follows the real install (drive/ProgramFiles can be
|
|
36
|
+
* relocated) instead of a hard-coded path.
|
|
37
|
+
* @returns {string[]}
|
|
38
|
+
*/
|
|
39
|
+
function chromeCandidates() {
|
|
40
|
+
const candidates = [
|
|
41
|
+
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
|
42
|
+
'/Applications/Chromium.app/Contents/MacOS/Chromium',
|
|
43
|
+
'/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
|
|
44
|
+
'/usr/bin/google-chrome',
|
|
45
|
+
'/usr/bin/google-chrome-stable',
|
|
46
|
+
'/usr/bin/chromium',
|
|
47
|
+
'/usr/bin/chromium-browser',
|
|
48
|
+
'/snap/bin/chromium',
|
|
49
|
+
]
|
|
50
|
+
if (process.platform === 'win32') {
|
|
51
|
+
const roots = [process.env.ProgramFiles, process.env['ProgramFiles(x86)'], process.env.ProgramW6432, process.env.LOCALAPPDATA]
|
|
52
|
+
.filter((root) => typeof root === 'string' && root !== '')
|
|
53
|
+
const relatives = [
|
|
54
|
+
['Google', 'Chrome', 'Application', 'chrome.exe'],
|
|
55
|
+
['Chromium', 'Application', 'chrome.exe'],
|
|
56
|
+
['Microsoft', 'Edge', 'Application', 'msedge.exe'],
|
|
57
|
+
]
|
|
58
|
+
for (const root of roots) {
|
|
59
|
+
for (const relative of relatives) candidates.push(path.join(root, ...relative))
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return candidates
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const DEFAULT_USER_AGENT =
|
|
66
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' +
|
|
67
|
+
'(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
|
|
68
|
+
|
|
69
|
+
/** Ali yun WAF challenge markers: page is not the product page we asked for. */
|
|
70
|
+
const WAF_HINT_RE = /aliyun|acw_sc__v2|_____tmd_____|验证码|captcha|challenge-platform|error code.*?5\d\d/i
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* True when the rendered HTML is a WAF challenge page rather than a product
|
|
74
|
+
* page. A page carrying `__NEXT_DATA__` is always the real one.
|
|
75
|
+
* @param {unknown} html
|
|
76
|
+
*/
|
|
77
|
+
export function looksLikeWafChallenge(html) {
|
|
78
|
+
if (typeof html !== 'string' || html === '') return false
|
|
79
|
+
if (html.includes('__NEXT_DATA__')) return false
|
|
80
|
+
return WAF_HINT_RE.test(html)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** One retry for transient failures; the domestic site challenges in bursts. */
|
|
84
|
+
const RETRY_DELAY_MS = 1500
|
|
85
|
+
const RETRYABLE_FAILURES = new Set(['szlcsc-waf-challenge', 'chrome-render-empty', 'page-parse-failed'])
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* How to fix a missing Chrome, phrased so an agent can act on it directly.
|
|
89
|
+
* @param {{ path?: string, source?: string, tried: string[], configPath?: string, envPath?: string }} info
|
|
90
|
+
* @returns {string}
|
|
91
|
+
*/
|
|
92
|
+
export function chromeHint(info) {
|
|
93
|
+
const parts = ['未找到可用的 Chrome/Chromium,国内站人民币价与国内库存不可用(已回退为 USD 估算)。']
|
|
94
|
+
if (info.configPath !== undefined || info.envPath !== undefined) {
|
|
95
|
+
const missing = []
|
|
96
|
+
if (info.configPath !== undefined) missing.push(`config.chromePath=${info.configPath}`)
|
|
97
|
+
if (info.envPath !== undefined) missing.push(`CHROME_BIN=${info.envPath}`)
|
|
98
|
+
parts.push(`当前指定的路径不存在:${missing.join(',')}。`)
|
|
99
|
+
}
|
|
100
|
+
parts.push(`已尝试 ${info.tried.length} 个常见路径${info.tried.length === 0 ? '' : `(${info.tried.slice(0, 5).join(' ; ')}${info.tried.length > 5 ? ' …' : ''})`}。`)
|
|
101
|
+
parts.push(
|
|
102
|
+
'请任选其一修复后重试:① 在本插件配置里加 chromePath: "<本机 Chrome 可执行文件绝对路径>"' +
|
|
103
|
+
'(profile 的 cordis.patch.yml,条目 id 为 lcsc-parts);② 给宿主进程设置环境变量 CHROME_BIN。' +
|
|
104
|
+
'定位方法:Windows `where chrome` 或查看 %ProgramFiles%\\Google\\Chrome\\Application\\chrome.exe;' +
|
|
105
|
+
'macOS `ls /Applications | grep -i chrome`;Linux `which google-chrome chromium`。',
|
|
106
|
+
)
|
|
107
|
+
return parts.join('')
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Locate a Chrome/Chromium binary and explain where it came from.
|
|
112
|
+
* Precedence: explicit config → CHROME_BIN → auto-discovery.
|
|
113
|
+
* @param {string} [explicit]
|
|
114
|
+
* @returns {{ path?: string, source?: 'config' | 'chrome_bin' | 'auto', tried: string[], configPath?: string, envPath?: string }}
|
|
115
|
+
*/
|
|
116
|
+
export function findChromeInfo(explicit) {
|
|
117
|
+
const configPath = typeof explicit === 'string' && explicit !== '' ? explicit : undefined
|
|
118
|
+
if (configPath !== undefined && fs.existsSync(configPath)) return { path: configPath, source: 'config', tried: [], configPath }
|
|
119
|
+
const envPath = typeof process.env.CHROME_BIN === 'string' && process.env.CHROME_BIN !== '' ? process.env.CHROME_BIN : undefined
|
|
120
|
+
if (envPath !== undefined && fs.existsSync(envPath)) return { path: envPath, source: 'chrome_bin', tried: [], envPath }
|
|
121
|
+
const tried = []
|
|
122
|
+
for (const candidate of chromeCandidates()) {
|
|
123
|
+
if (tried.includes(candidate)) continue
|
|
124
|
+
tried.push(candidate)
|
|
125
|
+
if (fs.existsSync(candidate)) return { path: candidate, source: 'auto', tried, configPath, envPath }
|
|
126
|
+
}
|
|
127
|
+
return { path: undefined, source: undefined, tried, configPath, envPath }
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Locate a Chrome/Chromium binary (compatibility wrapper around findChromeInfo). */
|
|
131
|
+
export function findChrome(explicit) {
|
|
132
|
+
return findChromeInfo(explicit).path
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Extract the `__NEXT_DATA__` JSON from an HTML document (or undefined). */
|
|
136
|
+
export function extractNextData(html) {
|
|
137
|
+
const match = String(html ?? '').match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/)
|
|
138
|
+
if (match === null) return undefined
|
|
139
|
+
try {
|
|
140
|
+
return JSON.parse(match[1])
|
|
141
|
+
} catch {
|
|
142
|
+
return undefined
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** Chrome profile subdirectories that are pure cache/garbage and can be deleted freely. */
|
|
147
|
+
const CHROME_PROFILE_JUNK = [
|
|
148
|
+
'Default/Cache',
|
|
149
|
+
'Default/Code Cache',
|
|
150
|
+
'Default/GPUCache',
|
|
151
|
+
'Default/DawnCache',
|
|
152
|
+
'Default/DawnGraphiteCache',
|
|
153
|
+
'Default/DawnWebGPUCache',
|
|
154
|
+
'Default/GraphiteDawnCache',
|
|
155
|
+
'Default/ShaderCache',
|
|
156
|
+
'Default/Service Worker/CacheStorage',
|
|
157
|
+
'Default/Service Worker/ScriptCache',
|
|
158
|
+
'Default/Media Cache',
|
|
159
|
+
'Default/Session Storage',
|
|
160
|
+
'Default/WebStorage',
|
|
161
|
+
'Crashpad',
|
|
162
|
+
'ShaderCache',
|
|
163
|
+
'GrShaderCache',
|
|
164
|
+
'BrowserMetrics',
|
|
165
|
+
'component_crx_cache',
|
|
166
|
+
'extensions_crx_cache',
|
|
167
|
+
]
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Delete cache-only subdirectories from a Chrome profile, keeping cookies and
|
|
171
|
+
* preferences (the WAF challenge cookie lives in the profile).
|
|
172
|
+
* @param {string} profileDir
|
|
173
|
+
*/
|
|
174
|
+
export function pruneChromeProfile(profileDir) {
|
|
175
|
+
for (const relative of CHROME_PROFILE_JUNK) {
|
|
176
|
+
try {
|
|
177
|
+
fs.rmSync(path.join(profileDir, relative), { recursive: true, force: true })
|
|
178
|
+
} catch {
|
|
179
|
+
// best effort
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Prune the JSON result cache: drop entries older than `ttlMs`, then keep at
|
|
186
|
+
* most `maxEntries` newest entries. This keeps `~/.cache/dsh-plugin-lcsc/cn`
|
|
187
|
+
* bounded instead of accumulating one file per part ever queried.
|
|
188
|
+
* @param {string} cacheDir
|
|
189
|
+
* @param {{ ttlMs?: number, maxEntries?: number }} [options]
|
|
190
|
+
* @returns {{ removed: number, kept: number }}
|
|
191
|
+
*/
|
|
192
|
+
export function pruneJsonCache(cacheDir, options = {}) {
|
|
193
|
+
const ttlMs = Number.isFinite(options.ttlMs) ? options.ttlMs : 24 * 60 * 60 * 1000
|
|
194
|
+
const maxEntries = Number.isFinite(options.maxEntries) ? Math.max(1, Math.floor(options.maxEntries)) : 800
|
|
195
|
+
let names
|
|
196
|
+
try {
|
|
197
|
+
names = fs.readdirSync(cacheDir).filter((name) => name.endsWith('.json'))
|
|
198
|
+
} catch {
|
|
199
|
+
return { removed: 0, kept: 0 }
|
|
200
|
+
}
|
|
201
|
+
const now = Date.now()
|
|
202
|
+
/** @type {{ file: string, fetchedAt: number }[]} */
|
|
203
|
+
const alive = []
|
|
204
|
+
let removed = 0
|
|
205
|
+
for (const name of names) {
|
|
206
|
+
const file = path.join(cacheDir, name)
|
|
207
|
+
let record
|
|
208
|
+
try {
|
|
209
|
+
record = JSON.parse(fs.readFileSync(file, 'utf8'))
|
|
210
|
+
} catch {
|
|
211
|
+
record = undefined
|
|
212
|
+
}
|
|
213
|
+
const fetchedAt = record === undefined ? Number.NaN : Date.parse(record.fetched_at)
|
|
214
|
+
if (!Number.isFinite(fetchedAt) || now - fetchedAt > ttlMs) {
|
|
215
|
+
try {
|
|
216
|
+
fs.rmSync(file, { force: true })
|
|
217
|
+
removed += 1
|
|
218
|
+
} catch {
|
|
219
|
+
// best effort
|
|
220
|
+
}
|
|
221
|
+
continue
|
|
222
|
+
}
|
|
223
|
+
alive.push({ file, fetchedAt })
|
|
224
|
+
}
|
|
225
|
+
if (alive.length > maxEntries) {
|
|
226
|
+
alive.sort((left, right) => left.fetchedAt - right.fetchedAt)
|
|
227
|
+
for (const entry of alive.slice(0, alive.length - maxEntries)) {
|
|
228
|
+
try {
|
|
229
|
+
fs.rmSync(entry.file, { force: true })
|
|
230
|
+
removed += 1
|
|
231
|
+
} catch {
|
|
232
|
+
// best effort
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
return { removed, kept: maxEntries }
|
|
236
|
+
}
|
|
237
|
+
return { removed, kept: alive.length }
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Parse a rendered `item.szlcsc.com` page into the official domestic record.
|
|
242
|
+
* @param {string} html
|
|
243
|
+
* @returns {{ price?: number, model?: string, brand?: string, package?: string, stock?: number, min_buy?: number, packing?: string, unit?: string, category?: string, desc?: string, product_id?: string } | undefined}
|
|
244
|
+
*/
|
|
245
|
+
export function parseItemHtml(html) {
|
|
246
|
+
const text = String(html ?? '')
|
|
247
|
+
const nextData = extractNextData(text)
|
|
248
|
+
if (nextData === undefined) {
|
|
249
|
+
// JSON-LD fallback for pages that omit __NEXT_DATA__.
|
|
250
|
+
const fallback = text.match(/"priceCurrency":"CNY","price":([0-9.]+)/)
|
|
251
|
+
if (fallback === null) return undefined
|
|
252
|
+
return { price: Number(fallback[1]) }
|
|
253
|
+
}
|
|
254
|
+
const pageProps = nextData?.props?.pageProps ?? {}
|
|
255
|
+
const webData = pageProps.webData ?? {}
|
|
256
|
+
const productRecord = webData.productRecord ?? {}
|
|
257
|
+
const price = Number(pageProps.price)
|
|
258
|
+
const result = {
|
|
259
|
+
price: Number.isFinite(price) ? price : undefined,
|
|
260
|
+
model: stringOrUndefined(productRecord.productModel),
|
|
261
|
+
brand: stringOrUndefined(productRecord.productGradePlateName),
|
|
262
|
+
package: stringOrUndefined(productRecord.encapsulationModel),
|
|
263
|
+
stock: integerOrUndefined(productRecord.stockNumber),
|
|
264
|
+
min_buy: integerOrUndefined(productRecord.minBuyNumber),
|
|
265
|
+
packing: stringOrUndefined(productRecord.productArrange),
|
|
266
|
+
unit: stringOrUndefined(productRecord.productUnit),
|
|
267
|
+
category: stringOrUndefined(productRecord.productType),
|
|
268
|
+
desc: stringOrUndefined(productRecord.productName) ?? stringOrUndefined(productRecord.remark),
|
|
269
|
+
product_id: productRecord.productId ?? productRecord.productCode,
|
|
270
|
+
}
|
|
271
|
+
if (result.price === undefined) {
|
|
272
|
+
const fallback = text.match(/"priceCurrency":"CNY","price":([0-9.]+)/)
|
|
273
|
+
if (fallback !== null) result.price = Number(fallback[1])
|
|
274
|
+
}
|
|
275
|
+
return result
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/** @param {unknown} value @returns {string | undefined} */
|
|
279
|
+
function stringOrUndefined(value) {
|
|
280
|
+
if (typeof value !== 'string') return undefined
|
|
281
|
+
const text = value.trim()
|
|
282
|
+
return text === '' ? undefined : text
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** @param {unknown} value @returns {number | undefined} */
|
|
286
|
+
function integerOrUndefined(value) {
|
|
287
|
+
const number = typeof value === 'string' ? Number(value) : value
|
|
288
|
+
return typeof number === 'number' && Number.isInteger(number) ? number : undefined
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** @param {number} ms */
|
|
292
|
+
function sleep(ms) {
|
|
293
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Create the domestic-CNY adapter.
|
|
298
|
+
* @param {{
|
|
299
|
+
* chromePath?: string,
|
|
300
|
+
* cacheDir?: string,
|
|
301
|
+
* cacheTtlMs?: number,
|
|
302
|
+
* timeoutMs?: number,
|
|
303
|
+
* concurrency?: number,
|
|
304
|
+
* browserIdleTtlMs?: number,
|
|
305
|
+
* verbose?: boolean,
|
|
306
|
+
* logger?: { warn?: (message: string) => void, info?: (message: string) => void },
|
|
307
|
+
* }} [options]
|
|
308
|
+
*/
|
|
309
|
+
export function createCnAdapter(options = {}) {
|
|
310
|
+
const chromeInfo = findChromeInfo(options.chromePath)
|
|
311
|
+
const chromePath = chromeInfo.path
|
|
312
|
+
const chromeHintText = chromePath === undefined ? chromeHint(chromeInfo) : undefined
|
|
313
|
+
const cacheDir =
|
|
314
|
+
typeof options.cacheDir === 'string' && options.cacheDir !== ''
|
|
315
|
+
? options.cacheDir
|
|
316
|
+
: path.join(os.homedir(), '.cache', 'dsh-plugin-lcsc', 'cn')
|
|
317
|
+
const profileDir = path.join(cacheDir, 'chrome-profile')
|
|
318
|
+
const cacheTtlMs = Number.isFinite(options.cacheTtlMs) ? options.cacheTtlMs : 24 * 60 * 60 * 1000
|
|
319
|
+
const timeoutMs = Number.isFinite(options.timeoutMs) ? options.timeoutMs : 30000
|
|
320
|
+
const concurrency = Math.max(1, Math.min(8, Math.floor(options.concurrency ?? 2)))
|
|
321
|
+
const browserIdleTtlMs = Number.isFinite(options.browserIdleTtlMs) ? options.browserIdleTtlMs : 120000
|
|
322
|
+
const cacheMaxEntries = Number.isFinite(options.cacheMaxEntries) ? Math.max(1, Math.floor(options.cacheMaxEntries)) : 800
|
|
323
|
+
const legacyProfileDir = path.join(cacheDir, 'legacy-profile')
|
|
324
|
+
let lastJsonPruneAt = 0
|
|
325
|
+
let queryCount = 0
|
|
326
|
+
const logger = options.logger
|
|
327
|
+
const memory = new Map()
|
|
328
|
+
let active = 0
|
|
329
|
+
/** @type {(() => void)[]} */
|
|
330
|
+
const waiting = []
|
|
331
|
+
|
|
332
|
+
/** Run `fn` under the adapter concurrency limit. */
|
|
333
|
+
async function withSlot(fn) {
|
|
334
|
+
while (active >= concurrency) {
|
|
335
|
+
await new Promise((resolve) => waiting.push(resolve))
|
|
336
|
+
}
|
|
337
|
+
active += 1
|
|
338
|
+
try {
|
|
339
|
+
return await fn()
|
|
340
|
+
} finally {
|
|
341
|
+
active -= 1
|
|
342
|
+
const next = waiting.shift()
|
|
343
|
+
if (next !== undefined) next()
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** HTTP GET JSON using the global fetch (host Node >= 20). */
|
|
348
|
+
async function getJson(url) {
|
|
349
|
+
const response = await fetch(url, {
|
|
350
|
+
headers: { 'User-Agent': DEFAULT_USER_AGENT, Accept: 'application/json' },
|
|
351
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
352
|
+
})
|
|
353
|
+
if (!response.ok) throw new Error(`HTTP ${response.status} for ${url}`)
|
|
354
|
+
return response.json()
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/* ------------------------- persistent Chrome (CDP) ----------------------- */
|
|
358
|
+
/**
|
|
359
|
+
* @typedef {{
|
|
360
|
+
* child: import('node:child_process').ChildProcess,
|
|
361
|
+
* write: NodeJS.WritableStream,
|
|
362
|
+
* read: NodeJS.ReadableStream,
|
|
363
|
+
* pending: Map<number, { resolve: (value: any) => void, reject: (reason: Error) => void, timer: NodeJS.Timeout }>,
|
|
364
|
+
* nextId: number,
|
|
365
|
+
* buffer: string,
|
|
366
|
+
* closed: boolean,
|
|
367
|
+
* }} BrowserState
|
|
368
|
+
*/
|
|
369
|
+
/** @type {BrowserState | undefined} */
|
|
370
|
+
let browser
|
|
371
|
+
/** @type {NodeJS.Timeout | undefined} */
|
|
372
|
+
let idleTimer
|
|
373
|
+
let cdpBroken = false
|
|
374
|
+
let cdpSessionErrors = 0
|
|
375
|
+
|
|
376
|
+
function shutdownBrowser() {
|
|
377
|
+
if (idleTimer !== undefined) {
|
|
378
|
+
clearTimeout(idleTimer)
|
|
379
|
+
idleTimer = undefined
|
|
380
|
+
}
|
|
381
|
+
const state = browser
|
|
382
|
+
browser = undefined
|
|
383
|
+
if (state === undefined) return
|
|
384
|
+
state.closed = true
|
|
385
|
+
for (const entry of state.pending.values()) {
|
|
386
|
+
clearTimeout(entry.timer)
|
|
387
|
+
entry.reject(new Error('chrome-shutdown'))
|
|
388
|
+
}
|
|
389
|
+
state.pending.clear()
|
|
390
|
+
try {
|
|
391
|
+
state.child.kill('SIGKILL')
|
|
392
|
+
} catch {
|
|
393
|
+
// already gone
|
|
394
|
+
}
|
|
395
|
+
// Keep cookies for the WAF challenge, drop HTTP/Code caches so the profile
|
|
396
|
+
// does not silently grow.
|
|
397
|
+
pruneChromeProfile(profileDir)
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
function touchIdle() {
|
|
401
|
+
if (idleTimer !== undefined) clearTimeout(idleTimer)
|
|
402
|
+
if (browserIdleTtlMs <= 0) return
|
|
403
|
+
idleTimer = setTimeout(() => shutdownBrowser(), browserIdleTtlMs)
|
|
404
|
+
idleTimer.unref?.()
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/** Start Chrome once and resolve when its DevTools pipe answers. */
|
|
408
|
+
async function startBrowser() {
|
|
409
|
+
if (browser !== undefined && browser.closed !== true) return browser
|
|
410
|
+
if (chromePath === undefined) throw new Error('chrome-not-found')
|
|
411
|
+
fs.mkdirSync(profileDir, { recursive: true })
|
|
412
|
+
// Clean caches left behind by an interrupted previous run.
|
|
413
|
+
pruneChromeProfile(profileDir)
|
|
414
|
+
const child = spawn(
|
|
415
|
+
chromePath,
|
|
416
|
+
[
|
|
417
|
+
'--headless=new',
|
|
418
|
+
'--disable-gpu',
|
|
419
|
+
'--no-first-run',
|
|
420
|
+
'--no-default-browser-check',
|
|
421
|
+
'--disable-background-networking',
|
|
422
|
+
'--disable-sync',
|
|
423
|
+
'--mute-audio',
|
|
424
|
+
'--disk-cache-size=52428800', // hard cap: 50 MiB HTTP cache
|
|
425
|
+
'--media-cache-size=10485760', // hard cap: 10 MiB media cache
|
|
426
|
+
'--remote-debugging-pipe',
|
|
427
|
+
`--user-data-dir=${profileDir}`,
|
|
428
|
+
'about:blank',
|
|
429
|
+
],
|
|
430
|
+
{ stdio: ['ignore', 'ignore', 'ignore', 'pipe', 'pipe'] },
|
|
431
|
+
)
|
|
432
|
+
const write = /** @type {NodeJS.WritableStream} */ (child.stdio[3])
|
|
433
|
+
const read = /** @type {NodeJS.ReadableStream} */ (child.stdio[4])
|
|
434
|
+
if (write === null || read === null) {
|
|
435
|
+
try {
|
|
436
|
+
child.kill('SIGKILL')
|
|
437
|
+
} catch {
|
|
438
|
+
// ignore
|
|
439
|
+
}
|
|
440
|
+
throw new Error('chrome-pipe-unavailable')
|
|
441
|
+
}
|
|
442
|
+
/** @type {BrowserState} */
|
|
443
|
+
const state = { child, write, read, pending: new Map(), nextId: 1, buffer: '', closed: false }
|
|
444
|
+
read.on('data', (chunk) => {
|
|
445
|
+
state.buffer += chunk.toString('utf8')
|
|
446
|
+
const parts = state.buffer.split('\u0000')
|
|
447
|
+
state.buffer = parts.pop() ?? ''
|
|
448
|
+
for (const part of parts) {
|
|
449
|
+
if (part.trim() === '') continue
|
|
450
|
+
let message
|
|
451
|
+
try {
|
|
452
|
+
message = JSON.parse(part)
|
|
453
|
+
} catch {
|
|
454
|
+
continue
|
|
455
|
+
}
|
|
456
|
+
if (message.id === undefined) continue
|
|
457
|
+
const entry = state.pending.get(message.id)
|
|
458
|
+
if (entry === undefined) continue
|
|
459
|
+
state.pending.delete(message.id)
|
|
460
|
+
clearTimeout(entry.timer)
|
|
461
|
+
if (message.error !== undefined) entry.reject(new Error(JSON.stringify(message.error)))
|
|
462
|
+
else entry.resolve(message.result ?? message)
|
|
463
|
+
}
|
|
464
|
+
})
|
|
465
|
+
child.on('error', (error) => {
|
|
466
|
+
state.closed = true
|
|
467
|
+
if (browser === state) browser = undefined
|
|
468
|
+
for (const entry of state.pending.values()) {
|
|
469
|
+
clearTimeout(entry.timer)
|
|
470
|
+
entry.reject(error instanceof Error ? error : new Error(String(error)))
|
|
471
|
+
}
|
|
472
|
+
state.pending.clear()
|
|
473
|
+
})
|
|
474
|
+
child.on('exit', () => {
|
|
475
|
+
state.closed = true
|
|
476
|
+
if (browser === state) browser = undefined
|
|
477
|
+
for (const entry of state.pending.values()) {
|
|
478
|
+
clearTimeout(entry.timer)
|
|
479
|
+
entry.reject(new Error('chrome-exited'))
|
|
480
|
+
}
|
|
481
|
+
state.pending.clear()
|
|
482
|
+
})
|
|
483
|
+
browser = state
|
|
484
|
+
// handshake
|
|
485
|
+
await cdpSend(state, 'Browser.getVersion', {}, undefined, 8000)
|
|
486
|
+
if (options.verbose === true) logger?.info?.(`[lcsc-cn] chrome started (pid ${child.pid})`)
|
|
487
|
+
return state
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* Send one CDP command over the pipe.
|
|
492
|
+
* @param {BrowserState} state
|
|
493
|
+
* @param {string} method
|
|
494
|
+
* @param {Record<string, unknown>} [params]
|
|
495
|
+
* @param {string} [sessionId]
|
|
496
|
+
* @param {number} [timeout]
|
|
497
|
+
*/
|
|
498
|
+
function cdpSend(state, method, params = {}, sessionId, timeout = 15000) {
|
|
499
|
+
return new Promise((resolve, reject) => {
|
|
500
|
+
if (state.closed) {
|
|
501
|
+
reject(new Error('chrome-closed'))
|
|
502
|
+
return
|
|
503
|
+
}
|
|
504
|
+
const id = state.nextId
|
|
505
|
+
state.nextId += 1
|
|
506
|
+
const timer = setTimeout(() => {
|
|
507
|
+
state.pending.delete(id)
|
|
508
|
+
reject(new Error(`cdp-timeout:${method}`))
|
|
509
|
+
}, timeout)
|
|
510
|
+
timer.unref?.()
|
|
511
|
+
state.pending.set(id, { resolve, reject, timer })
|
|
512
|
+
const payload = { id, method, params }
|
|
513
|
+
if (sessionId !== undefined) payload.sessionId = sessionId
|
|
514
|
+
state.write.write(`${JSON.stringify(payload)}\u0000`)
|
|
515
|
+
})
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/** Open `url` in a new tab, wait for the rendered DOM, close the tab. */
|
|
519
|
+
async function renderViaCdp(url, timeout = timeoutMs) {
|
|
520
|
+
const state = await startBrowser()
|
|
521
|
+
const deadline = Date.now() + timeout
|
|
522
|
+
let targetId
|
|
523
|
+
let sessionId
|
|
524
|
+
try {
|
|
525
|
+
const created = await cdpSend(state, 'Target.createTarget', { url: 'about:blank' }, undefined, 10000)
|
|
526
|
+
targetId = created.targetId
|
|
527
|
+
const attached = await cdpSend(state, 'Target.attachToTarget', { targetId, flatten: true }, undefined, 10000)
|
|
528
|
+
sessionId = attached.sessionId
|
|
529
|
+
await cdpSend(state, 'Page.enable', {}, sessionId, 5000)
|
|
530
|
+
await cdpSend(state, 'Runtime.enable', {}, sessionId, 5000)
|
|
531
|
+
await cdpSend(state, 'Page.navigate', { url }, sessionId, 10000)
|
|
532
|
+
let html = ''
|
|
533
|
+
while (Date.now() < deadline) {
|
|
534
|
+
await sleep(150)
|
|
535
|
+
try {
|
|
536
|
+
const evaluated = await cdpSend(
|
|
537
|
+
state,
|
|
538
|
+
'Runtime.evaluate',
|
|
539
|
+
{ expression: 'document.documentElement.outerHTML', returnByValue: true },
|
|
540
|
+
sessionId,
|
|
541
|
+
5000,
|
|
542
|
+
)
|
|
543
|
+
html = typeof evaluated?.result?.value === 'string' ? evaluated.result.value : ''
|
|
544
|
+
} catch {
|
|
545
|
+
// navigation might still be in flight; retry until the deadline
|
|
546
|
+
}
|
|
547
|
+
if (html.includes('__NEXT_DATA__') && /"price"\s*:/.test(html)) return html
|
|
548
|
+
}
|
|
549
|
+
return html
|
|
550
|
+
} finally {
|
|
551
|
+
if (sessionId !== undefined && state.closed !== true) {
|
|
552
|
+
try {
|
|
553
|
+
await cdpSend(state, 'Target.detachFromTarget', { sessionId }, undefined, 3000)
|
|
554
|
+
} catch {
|
|
555
|
+
// ignore
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
if (targetId !== undefined && state.closed !== true) {
|
|
559
|
+
try {
|
|
560
|
+
await cdpSend(state, 'Target.closeTarget', { targetId }, undefined, 3000)
|
|
561
|
+
} catch {
|
|
562
|
+
// ignore
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
touchIdle()
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/* ------------------- legacy per-page renderer (fallback) ----------------- */
|
|
570
|
+
/** @param {string} url @param {number} timeout @returns {Promise<string>} */
|
|
571
|
+
function renderViaDumpDom(url, timeout = timeoutMs) {
|
|
572
|
+
if (chromePath === undefined) return Promise.resolve('')
|
|
573
|
+
return new Promise((resolve) => {
|
|
574
|
+
fs.mkdirSync(legacyProfileDir, { recursive: true })
|
|
575
|
+
pruneChromeProfile(legacyProfileDir)
|
|
576
|
+
const args = [
|
|
577
|
+
'--headless=new',
|
|
578
|
+
'--disable-gpu',
|
|
579
|
+
'--no-first-run',
|
|
580
|
+
'--no-default-browser-check',
|
|
581
|
+
'--disable-background-networking',
|
|
582
|
+
'--disable-sync',
|
|
583
|
+
'--mute-audio',
|
|
584
|
+
'--disk-cache-size=52428800',
|
|
585
|
+
'--media-cache-size=10485760',
|
|
586
|
+
`--user-data-dir=${legacyProfileDir}`,
|
|
587
|
+
'--dump-dom',
|
|
588
|
+
'--virtual-time-budget=6000',
|
|
589
|
+
url,
|
|
590
|
+
]
|
|
591
|
+
let finished = false
|
|
592
|
+
let stdout = ''
|
|
593
|
+
const child = spawn(chromePath, args, { stdio: ['ignore', 'pipe', 'ignore'] })
|
|
594
|
+
const timer = setTimeout(() => {
|
|
595
|
+
if (finished) return
|
|
596
|
+
finished = true
|
|
597
|
+
try {
|
|
598
|
+
child.kill('SIGKILL')
|
|
599
|
+
} catch {
|
|
600
|
+
// ignore
|
|
601
|
+
}
|
|
602
|
+
resolve(stdout)
|
|
603
|
+
}, timeout)
|
|
604
|
+
child.stdout?.on('data', (chunk) => {
|
|
605
|
+
stdout += chunk.toString()
|
|
606
|
+
})
|
|
607
|
+
child.on('error', () => {
|
|
608
|
+
if (finished) return
|
|
609
|
+
finished = true
|
|
610
|
+
clearTimeout(timer)
|
|
611
|
+
resolve('')
|
|
612
|
+
})
|
|
613
|
+
child.on('close', () => {
|
|
614
|
+
if (finished) return
|
|
615
|
+
finished = true
|
|
616
|
+
clearTimeout(timer)
|
|
617
|
+
pruneChromeProfile(legacyProfileDir)
|
|
618
|
+
resolve(stdout)
|
|
619
|
+
})
|
|
620
|
+
})
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* Render a URL to DOM: persistent CDP browser first, dump-dom fallback.
|
|
625
|
+
* @param {string} url
|
|
626
|
+
* @param {number} [timeout]
|
|
627
|
+
* @returns {Promise<string>}
|
|
628
|
+
*/
|
|
629
|
+
async function renderHtml(url, timeout = timeoutMs) {
|
|
630
|
+
if (chromePath === undefined) return ''
|
|
631
|
+
if (!cdpBroken && cdpSessionErrors < 3) {
|
|
632
|
+
try {
|
|
633
|
+
return await renderViaCdp(url, timeout)
|
|
634
|
+
} catch (error) {
|
|
635
|
+
cdpSessionErrors += 1
|
|
636
|
+
if (options.verbose === true) logger?.warn?.(`[lcsc-cn] cdp render failed: ${error instanceof Error ? error.message : String(error)}`)
|
|
637
|
+
shutdownBrowser()
|
|
638
|
+
if (cdpSessionErrors >= 3) cdpBroken = true
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
return renderViaDumpDom(url, timeout)
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
/**
|
|
645
|
+
* Resolve the domestic product id for an LCSC code.
|
|
646
|
+
* @param {string} code
|
|
647
|
+
* @returns {Promise<string | undefined>}
|
|
648
|
+
*/
|
|
649
|
+
async function resolvePid(code) {
|
|
650
|
+
for (const base of [EASYEDA_URL, LCEDA_URL]) {
|
|
651
|
+
try {
|
|
652
|
+
const payload = await getJson(`${base}${encodeURIComponent(code)}/components`)
|
|
653
|
+
const result = payload?.result ?? {}
|
|
654
|
+
const pid = result?.szlcsc?.id ?? result?.lcsc?.id
|
|
655
|
+
if (pid !== undefined && pid !== null && String(pid) !== '') return String(pid)
|
|
656
|
+
} catch {
|
|
657
|
+
// try the next API
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
return undefined
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
/** @param {string} code */
|
|
664
|
+
function cachePath(code) {
|
|
665
|
+
return path.join(cacheDir, `${code}.json`)
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
/** @param {string} code */
|
|
669
|
+
function readCache(code) {
|
|
670
|
+
const inMemory = memory.get(code)
|
|
671
|
+
if (inMemory !== undefined && Date.now() - inMemory.ts < cacheTtlMs) {
|
|
672
|
+
return { ...inMemory.value, cached: true, cache_age_s: Math.round((Date.now() - inMemory.ts) / 1000) }
|
|
673
|
+
}
|
|
674
|
+
try {
|
|
675
|
+
const record = JSON.parse(fs.readFileSync(cachePath(code), 'utf8'))
|
|
676
|
+
const fetchedAt = Date.parse(record.fetched_at)
|
|
677
|
+
if (Number.isFinite(fetchedAt) && Date.now() - fetchedAt < cacheTtlMs) {
|
|
678
|
+
return { ...record, cached: true, cache_age_s: Math.round((Date.now() - fetchedAt) / 1000) }
|
|
679
|
+
}
|
|
680
|
+
} catch {
|
|
681
|
+
// cache miss
|
|
682
|
+
}
|
|
683
|
+
return undefined
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
/** @param {string} code @param {Record<string, unknown>} record */
|
|
687
|
+
function writeCache(code, record) {
|
|
688
|
+
try {
|
|
689
|
+
fs.mkdirSync(cacheDir, { recursive: true })
|
|
690
|
+
fs.writeFileSync(cachePath(code), JSON.stringify(record, null, 1), 'utf8')
|
|
691
|
+
} catch {
|
|
692
|
+
// cache write is best effort
|
|
693
|
+
}
|
|
694
|
+
memory.set(code, { ts: Date.now(), value: record })
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
/**
|
|
698
|
+
* Query one LCSC code. Never throws; returns `{ok:false, reason}` on failure.
|
|
699
|
+
* @param {string} code
|
|
700
|
+
* @param {{ refresh?: boolean }} [callOptions]
|
|
701
|
+
*/
|
|
702
|
+
async function query(code, callOptions = {}) {
|
|
703
|
+
const normalized = String(code ?? '').trim().toUpperCase()
|
|
704
|
+
if (normalized === '') return { ok: false, reason: 'empty-code' }
|
|
705
|
+
if (chromePath === undefined) {
|
|
706
|
+
return { ok: false, reason: 'chrome-not-found', tried: chromeInfo.tried, hint: chromeHintText }
|
|
707
|
+
}
|
|
708
|
+
queryCount += 1
|
|
709
|
+
// Prune expired JSON cache entries lazily (about once per hour / 50 hits).
|
|
710
|
+
if (queryCount === 1 || Date.now() - lastJsonPruneAt > 60 * 60 * 1000 || queryCount % 50 === 0) {
|
|
711
|
+
lastJsonPruneAt = Date.now()
|
|
712
|
+
pruneJsonCache(cacheDir, { ttlMs: cacheTtlMs, maxEntries: cacheMaxEntries })
|
|
713
|
+
}
|
|
714
|
+
const cached = callOptions.refresh === true ? undefined : readCache(normalized)
|
|
715
|
+
if (cached !== undefined && cached.ok !== false) {
|
|
716
|
+
touchIdle()
|
|
717
|
+
return cached
|
|
718
|
+
}
|
|
719
|
+
return withSlot(async () => {
|
|
720
|
+
/** @type {{ ok: false, reason: string, attempts: number } | undefined} */
|
|
721
|
+
let failure
|
|
722
|
+
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
723
|
+
if (attempt > 1) await sleep(RETRY_DELAY_MS)
|
|
724
|
+
try {
|
|
725
|
+
const pid = await resolvePid(normalized)
|
|
726
|
+
if (pid === undefined) return { ok: false, reason: 'product-id-not-found', attempts: attempt }
|
|
727
|
+
const html = await renderHtml(`${ITEM_URL}${pid}.html`)
|
|
728
|
+
if (html === '') {
|
|
729
|
+
failure = { ok: false, reason: 'chrome-render-empty', attempts: attempt }
|
|
730
|
+
continue
|
|
731
|
+
}
|
|
732
|
+
if (looksLikeWafChallenge(html)) {
|
|
733
|
+
failure = { ok: false, reason: 'szlcsc-waf-challenge', attempts: attempt }
|
|
734
|
+
continue
|
|
735
|
+
}
|
|
736
|
+
const parsed = parseItemHtml(html)
|
|
737
|
+
if (parsed === undefined || parsed.price === undefined) {
|
|
738
|
+
failure = { ok: false, reason: 'page-parse-failed', attempts: attempt }
|
|
739
|
+
continue
|
|
740
|
+
}
|
|
741
|
+
const record = {
|
|
742
|
+
ok: true,
|
|
743
|
+
lcsc: normalized,
|
|
744
|
+
product_id: pid,
|
|
745
|
+
unit: parsed.price,
|
|
746
|
+
currency: 'CNY',
|
|
747
|
+
stock: parsed.stock,
|
|
748
|
+
model: parsed.model,
|
|
749
|
+
brand: parsed.brand,
|
|
750
|
+
package: parsed.package,
|
|
751
|
+
min_buy: parsed.min_buy,
|
|
752
|
+
packing: parsed.packing,
|
|
753
|
+
unit_name: parsed.unit,
|
|
754
|
+
category: parsed.category,
|
|
755
|
+
desc: parsed.desc,
|
|
756
|
+
source: 'lcsc_cn_render',
|
|
757
|
+
fetched_at: new Date().toISOString(),
|
|
758
|
+
cache_age_s: 0,
|
|
759
|
+
cached: false,
|
|
760
|
+
}
|
|
761
|
+
writeCache(normalized, record)
|
|
762
|
+
return record
|
|
763
|
+
} catch (error) {
|
|
764
|
+
const reason = error instanceof Error ? error.message : String(error)
|
|
765
|
+
failure = { ok: false, reason, attempts: attempt }
|
|
766
|
+
if (!RETRYABLE_FAILURES.has(reason)) return failure
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
return failure ?? { ok: false, reason: 'cn-query-failed', attempts: 2 }
|
|
770
|
+
})
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
return {
|
|
774
|
+
chromePath,
|
|
775
|
+
/** Where the binary came from and what was tried; used by the tools' hints. */
|
|
776
|
+
chrome: { ...chromeInfo, hint: chromeHintText },
|
|
777
|
+
available: chromePath !== undefined,
|
|
778
|
+
cacheDir,
|
|
779
|
+
query,
|
|
780
|
+
resolvePid,
|
|
781
|
+
renderHtml,
|
|
782
|
+
clearMemory() {
|
|
783
|
+
memory.clear()
|
|
784
|
+
},
|
|
785
|
+
/** Remove this adapter's on-disk caches (JSON results + Chrome profiles). */
|
|
786
|
+
clearCache() {
|
|
787
|
+
shutdownBrowser()
|
|
788
|
+
try {
|
|
789
|
+
fs.rmSync(cacheDir, { recursive: true, force: true })
|
|
790
|
+
} catch {
|
|
791
|
+
// best effort
|
|
792
|
+
}
|
|
793
|
+
memory.clear()
|
|
794
|
+
},
|
|
795
|
+
pruneCaches() {
|
|
796
|
+
const json = pruneJsonCache(cacheDir, { ttlMs: cacheTtlMs, maxEntries: cacheMaxEntries })
|
|
797
|
+
pruneChromeProfile(profileDir)
|
|
798
|
+
pruneChromeProfile(legacyProfileDir)
|
|
799
|
+
return json
|
|
800
|
+
},
|
|
801
|
+
dispose() {
|
|
802
|
+
shutdownBrowser()
|
|
803
|
+
pruneChromeProfile(profileDir)
|
|
804
|
+
pruneChromeProfile(legacyProfileDir)
|
|
805
|
+
pruneJsonCache(cacheDir, { ttlMs: cacheTtlMs, maxEntries: cacheMaxEntries })
|
|
806
|
+
},
|
|
807
|
+
}
|
|
808
|
+
}
|