@jackwener/opencli 1.7.14 → 1.7.15

package/clis/dianping/cityResolver.js

@@ -0,0 +1,185 @@
+/**
+ * Async city → cityId resolver for dianping adapters.
+ *
+ * Wraps the synchronous static-map `resolveCityId` from utils.js and falls
+ * back to a live lookup against www.dianping.com when the input is not in
+ * the curated map. Resolves both pinyin slugs (e.g. "shantou") and Chinese
+ * names (e.g. "汕头") by reading dianping itself, so the adapter no longer
+ * has to ship a complete static city table.
+ *
+ * Strategy:
+ *   1. Empty / null  → null  (let the cookie's default city stand).
+ *   2. All-digits    → numeric cityId pass-through.
+ *   3. Static map    → fast path, no network. Reuses utils.CITY_ID.
+ *   4. Pinyin slug   → goto https://www.dianping.com/<slug>, parse the
+ *                       cityId out of any /search/keyword/{id}/ link.
+ *   5. Chinese name  → goto https://www.dianping.com/citylist, build a
+ *                       Chinese-name → pinyin map, then resolve the slug
+ *                       as in step 4.
+ *
+ * Resolved (input → cityId) pairs are memoized in a module-level Map so a
+ * second search in the same process skips both navigations.
+ */
+
+import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors';
+import { CITY_ID, resolveCityId } from './utils.js';
+
+const CHINESE_RE = /^[一-龥]+$/;
+const PINYIN_RE = /^[a-z]+$/;
+
+const RESOLVE_CACHE = new Map();
+
+/**
+ * Reset the in-process resolver cache. Exposed for tests so each case
+ * starts from a clean slate; production code never needs to call this.
+ */
+export function clearCityResolverCache() {
+    RESOLVE_CACHE.clear();
+}
+
+/**
+ * Async resolver. Falls back to live dianping pages only when the input
+ * is not in the static map.
+ *
+ * @param {{ goto: Function, evaluate: Function }} page  page handle from the adapter func
+ * @param {string|number|null|undefined} cityArg         user-supplied city (name, pinyin, or numeric id)
+ * @returns {Promise<number|null>}                       numeric cityId, or null to use the cookie default
+ */
+export async function resolveCityIdAsync(page, cityArg) {
+    if (cityArg == null || cityArg === '') return null;
+    const raw = String(cityArg).trim();
+    if (!raw) return null;
+    if (/^\d+$/.test(raw)) return Number(raw);
+
+    const lowered = raw.toLowerCase();
+
+    // Fast path: reuse the synchronous static map. resolveCityId throws
+    // ArgumentError when the input is unknown — that's the trigger to fall
+    // back to dynamic resolution rather than surface the error to the user.
+    try {
+        const staticId = resolveCityId(raw);
+        if (staticId != null) return staticId;
+    } catch (err) {
+        if (err?.code !== 'ARGUMENT') throw err;
+    }
+
+    if (RESOLVE_CACHE.has(lowered)) return RESOLVE_CACHE.get(lowered);
+    if (RESOLVE_CACHE.has(raw)) return RESOLVE_CACHE.get(raw);
+
+    let pinyin = null;
+    if (PINYIN_RE.test(lowered)) {
+        pinyin = lowered;
+    } else if (CHINESE_RE.test(raw)) {
+        pinyin = await lookupPinyinFromCitylist(page, raw);
+        if (!pinyin) {
+            const known = Object.keys(CITY_ID).filter((k) => /^[a-z]+$/.test(k)).join(', ');
+            throw new ArgumentError(
+                'city',
+                `unknown city '${cityArg}'. pass a numeric cityId, a pinyin slug (e.g. shantou), `
+                + `a Chinese name listed on dianping.com/citylist, or one of: ${known}`,
+            );
+        }
+    } else {
+        const known = Object.keys(CITY_ID).filter((k) => /^[a-z]+$/.test(k)).join(', ');
+        throw new ArgumentError(
+            'city',
+            `unknown city '${cityArg}'. pass a numeric cityId, a pinyin slug (e.g. shantou), `
+            + `a Chinese name listed on dianping.com/citylist, or one of: ${known}`,
+        );
+    }
+
+    const cityId = await fetchCityIdByPinyin(page, pinyin);
+    if (!cityId) {
+        throw new CommandExecutionError(
+            `dianping could not resolve cityId for '${cityArg}' (pinyin=${pinyin}); `
+            + `the city page rendered without a /search/keyword/{id}/ link`,
+        );
+    }
+
+    RESOLVE_CACHE.set(lowered, cityId);
+    RESOLVE_CACHE.set(pinyin, cityId);
+    if (CHINESE_RE.test(raw)) RESOLVE_CACHE.set(raw, cityId);
+    return cityId;
+}
+
+/**
+ * Read https://www.dianping.com/citylist and return a Chinese-name → pinyin
+ * slug map for every city link present on the page. Used when the user
+ * supplied a Chinese name that isn't in the static map.
+ */
+async function lookupPinyinFromCitylist(page, chineseName) {
+    await page.goto('https://www.dianping.com/citylist');
+    const map = await page.evaluate(`(${buildCitylistMap.toString()})()`);
+    if (!map || typeof map !== 'object' || Object.keys(map).length === 0) {
+        throw new CommandExecutionError(
+            'dianping citylist did not render any city anchors; cannot resolve Chinese city names',
+        );
+    }
+    if (map && typeof map === 'object' && map[chineseName]) {
+        return String(map[chineseName]).toLowerCase();
+    }
+    return null;
+}
+
+/**
+ * Pure DOM extractor for /citylist. Walks every anchor on the page and
+ * keeps the ones whose href matches the per-city slug shape and whose
+ * text is a pure-Chinese label. Defined at module scope so the same code
+ * can be exercised from JSDOM tests via toString() injection.
+ */
+export function buildCitylistMap() {
+    const map = {};
+    const anchors = document.querySelectorAll('a');
+    anchors.forEach((a) => {
+        const hrefRaw = a.getAttribute('href') || '';
+        const text = ((a.textContent || '').trim());
+        if (!text || !/^[一-龥]+$/.test(text)) return;
+        const href = hrefRaw.replace(/^https?:/, '');
+        const m = href.match(/^\/\/(?:www\.)?dianping\.com\/([a-z]+)\/?$/)
+            || href.match(/^\/([a-z]+)\/?$/);
+        if (!m) return;
+        const slug = m[1].toLowerCase();
+        // Filter out non-city slugs that share the shape (e.g. /citylist itself,
+        // /promo, /events). Only register the first slug per Chinese label.
+        if (slug === 'citylist' || slug === 'promo' || slug === 'events') return;
+        if (!map[text]) map[text] = slug;
+    });
+    return map;
+}
+
+/**
+ * Visit https://www.dianping.com/<slug> and pull the cityId out of any
+ * /search/keyword/{id}/ anchor. The PC city landing page renders these
+ * links server-side for every category card, so a single goto + DOM read
+ * is enough — no extra clicks or hydration wait.
+ */
+async function fetchCityIdByPinyin(page, pinyin) {
+    await page.goto(`https://www.dianping.com/${pinyin}`);
+    const cityId = await page.evaluate(`(${extractCityIdFromPage.toString()})()`);
+    return Number.isInteger(cityId) && cityId > 0 ? cityId : null;
+}
+
+/**
+ * Pure DOM extractor for the per-city landing page. Defined at module
+ * scope so the same code is exercised in JSDOM tests via toString().
+ */
+export function extractCityIdFromPage() {
+    const baseHref = (typeof location !== 'undefined' && location.href) || 'https://www.dianping.com/';
+    const anchors = document.querySelectorAll('a[href]');
+    for (const a of anchors) {
+        const hrefRaw = a.getAttribute('href') || '';
+        let url;
+        try {
+            url = new URL(hrefRaw, baseHref);
+        } catch {
+            continue;
+        }
+        if (url.protocol !== 'https:') continue;
+        if (url.hostname !== 'www.dianping.com' && url.hostname !== 'dianping.com') continue;
+        const m = url.pathname.match(/^\/search\/keyword\/(\d+)(?:\/|$)/);
+        if (!m) continue;
+        const n = Number(m[1]);
+        if (Number.isInteger(n) && n > 0) return n;
+    }
+    return null;
+}

package/clis/dianping/dianping.test.js

@@ -22,6 +22,12 @@ import {
 } from './utils.js';
 import { extractSearchRows } from './search.js';
 import { extractShopFields } from './shop.js';
+import {
+    buildCitylistMap,
+    clearCityResolverCache,
+    extractCityIdFromPage,
+    resolveCityIdAsync,
+} from './cityResolver.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const SHOP_FIXTURE = readFileSync(join(__dirname, '__fixtures__/shop.html'), 'utf8');
@@ -104,6 +110,154 @@ describe('dianping adapter — helpers', () => {
     });
 });
 
+describe('dianping adapter — async city resolver', () => {
+    beforeEach(() => {
+        clearCityResolverCache();
+    });
+
+    it('returns null/numeric/static-map ids without ever touching the page', async () => {
+        const page = createPageMock({});
+
+        expect(await resolveCityIdAsync(page, undefined)).toBeNull();
+        expect(await resolveCityIdAsync(page, '')).toBeNull();
+        expect(await resolveCityIdAsync(page, '   ')).toBeNull();
+        expect(await resolveCityIdAsync(page, 47)).toBe(47);
+        expect(await resolveCityIdAsync(page, '47')).toBe(47);
+        expect(await resolveCityIdAsync(page, '北京')).toBe(2);
+        expect(await resolveCityIdAsync(page, 'shanghai')).toBe(1);
+
+        expect(page.goto).not.toHaveBeenCalled();
+        expect(page.evaluate).not.toHaveBeenCalled();
+    });
+
+    it('falls back to /<pinyin> for an unknown lowercase slug, then caches', async () => {
+        const goto = vi.fn().mockResolvedValue(undefined);
+        const evaluate = vi.fn().mockResolvedValue(207);
+        const page = { goto, evaluate, wait: vi.fn() };
+
+        expect(await resolveCityIdAsync(page, 'shantou')).toBe(207);
+        expect(goto).toHaveBeenCalledTimes(1);
+        expect(goto).toHaveBeenCalledWith('https://www.dianping.com/shantou');
+
+        // Second call hits the in-process cache, no extra navigation.
+        expect(await resolveCityIdAsync(page, 'shantou')).toBe(207);
+        expect(goto).toHaveBeenCalledTimes(1);
+    });
+
+    it('resolves a Chinese name via /citylist + /<pinyin>, then caches both forms', async () => {
+        const goto = vi.fn().mockResolvedValue(undefined);
+        const evaluate = vi.fn()
+            .mockResolvedValueOnce({ '汕头': 'shantou', '佛山': 'foshan' })
+            .mockResolvedValueOnce(207);
+        const page = { goto, evaluate, wait: vi.fn() };
+
+        expect(await resolveCityIdAsync(page, '汕头')).toBe(207);
+        expect(goto).toHaveBeenNthCalledWith(1, 'https://www.dianping.com/citylist');
+        expect(goto).toHaveBeenNthCalledWith(2, 'https://www.dianping.com/shantou');
+
+        // Cached for the Chinese name AND the discovered pinyin.
+        expect(await resolveCityIdAsync(page, '汕头')).toBe(207);
+        expect(await resolveCityIdAsync(page, 'shantou')).toBe(207);
+        expect(goto).toHaveBeenCalledTimes(2);
+    });
+
+    it('rejects mixed/garbage input with ArgumentError before any navigation', async () => {
+        const page = createPageMock({});
+        await expect(resolveCityIdAsync(page, 'not-a-city!')).rejects.toThrow(ArgumentError);
+        expect(page.goto).not.toHaveBeenCalled();
+    });
+
+    it('rejects a Chinese name that is not on /citylist with ArgumentError', async () => {
+        const goto = vi.fn().mockResolvedValue(undefined);
+        const evaluate = vi.fn().mockResolvedValueOnce({ '北京': 'beijing' });
+        const page = { goto, evaluate, wait: vi.fn() };
+
+        await expect(resolveCityIdAsync(page, '某虚构城')).rejects.toThrow(ArgumentError);
+        expect(goto).toHaveBeenCalledTimes(1);
+        expect(goto).toHaveBeenCalledWith('https://www.dianping.com/citylist');
+    });
+
+    it('throws CommandExecutionError when citylist renders without city anchors', async () => {
+        const goto = vi.fn().mockResolvedValue(undefined);
+        const evaluate = vi.fn().mockResolvedValueOnce({});
+        const page = { goto, evaluate, wait: vi.fn() };
+
+        await expect(resolveCityIdAsync(page, '汕头')).rejects.toThrow(CommandExecutionError);
+        expect(goto).toHaveBeenCalledTimes(1);
+        expect(goto).toHaveBeenCalledWith('https://www.dianping.com/citylist');
+    });
+
+    it('throws CommandExecutionError when the per-city page lacks a /search/keyword/{id}/ link', async () => {
+        const goto = vi.fn().mockResolvedValue(undefined);
+        const evaluate = vi.fn().mockResolvedValueOnce(null);
+        const page = { goto, evaluate, wait: vi.fn() };
+
+        await expect(resolveCityIdAsync(page, 'newcity')).rejects.toThrow(CommandExecutionError);
+    });
+
+    it('buildCitylistMap keeps Chinese-labeled city slugs and drops non-city paths', () => {
+        const dom = new JSDOM(`
+            <html><body>
+                <a href="//www.dianping.com/shanghai">上海</a>
+                <a href="//www.dianping.com/shantou">汕头</a>
+                <a href="/beijing">北京</a>
+                <a href="//www.dianping.com/citylist">更多城市 ></a>
+                <a href="//www.dianping.com/promo">优惠</a>
+                <a href="//www.dianping.com/shanghai">上海</a>
+                <a href="https://www.dianping.com/member/123">可乐不加冰</a>
+                <a href="https://example.com/notacity">东京</a>
+            </body></html>
+        `);
+        globalThis.document = dom.window.document;
+
+        try {
+            const map = buildCitylistMap();
+            expect(map['上海']).toBe('shanghai');
+            expect(map['汕头']).toBe('shantou');
+            expect(map['北京']).toBe('beijing');
+            expect(map['更多城市 >']).toBeUndefined();
+            expect(map['优惠']).toBeUndefined();
+            expect(map['可乐不加冰']).toBeUndefined();
+            expect(map['东京']).toBeUndefined();
+        } finally {
+            delete globalThis.document;
+        }
+    });
+
+    it('extractCityIdFromPage pulls the cityId from the first /search/keyword/{id}/ link', () => {
+        const dom = new JSDOM(`
+            <html><body>
+                <script>window.bad = "/search/keyword/999/";</script>
+                <a href="https://example.com/search/keyword/888/0_x">wrong host</a>
+                <a href="https://www.dianping.com.evil.com/search/keyword/666/0_x">host suffix</a>
+                <a href="http://www.dianping.com/search/keyword/777/0_x">non-https</a>
+                <a href="/search/keyword/207/0_%E5%88%BA%E8%BA%AB">刺身</a>
+                <a href="/search/category/207/10">美食</a>
+            </body></html>
+        `, { url: 'https://www.dianping.com/shantou' });
+        globalThis.document = dom.window.document;
+        globalThis.location = dom.window.location;
+
+        try {
+            expect(extractCityIdFromPage()).toBe(207);
+        } finally {
+            delete globalThis.document;
+            delete globalThis.location;
+        }
+    });
+
+    it('extractCityIdFromPage returns null when no /search/keyword/{id}/ link exists', () => {
+        const dom = new JSDOM(`<html><body><main>blocked</main></body></html>`);
+        globalThis.document = dom.window.document;
+
+        try {
+            expect(extractCityIdFromPage()).toBeNull();
+        } finally {
+            delete globalThis.document;
+        }
+    });
+});
+
 describe('dianping adapter — search runtime', () => {
     const command = getRegistry().get('dianping/search');
 

package/clis/dianping/search.js

@@ -19,9 +19,9 @@ import {
     parsePrice,
     parseReviewCount,
     requireSearchLimit,
-    resolveCityId,
     wrapDianpingStep,
 } from './utils.js';
+import { resolveCityIdAsync } from './cityResolver.js';
 
 /**
  * Pure DOM extractor for the dianping search-results page.
@@ -98,7 +98,7 @@ cli({
     strategy: Strategy.COOKIE,
     args: [
         { name: 'keyword', required: true, positional: true, help: '搜索关键词，例如 "火锅"' },
-        { name: 'city', help: '城市名（北京/上海/beijing/...）或 cityId 数字。不传则使用 cookie 默认城市' },
+        { name: 'city', help: '城市名（北京/上海/汕头/beijing/shantou/...）或 cityId 数字。未在静态表中的城市会通过 dianping.com 在线解析。不传则使用 cookie 默认城市' },
         { name: 'limit', type: 'int', default: 15, help: '返回的店铺数量（最多 15，dianping 单页固定 15 条）' },
     ],
     columns: SEARCH_COLUMNS,
@@ -108,7 +108,10 @@ cli({
 
         const limit = requireSearchLimit(kwargs.limit);
 
-        const cityId = resolveCityId(kwargs.city);
+        const cityId = await wrapDianpingStep(
+            'city resolve',
+            () => resolveCityIdAsync(page, kwargs.city),
+        );
         const path = cityId
             ? `/search/keyword/${cityId}/0_${encodeURIComponent(keyword)}`
             : `/search/keyword/0/0_${encodeURIComponent(keyword)}`;

package/clis/douyin/_shared/browser-fetch.js

@@ -15,10 +15,22 @@ export async function browserFetch(page, method, url, options = {}) {
         },
         ${options.body ? `body: JSON.stringify(${JSON.stringify(options.body)}),` : ''}
       });
-      return res.json();
+      const text = await res.text();
+      if (!text) return null;
+      return JSON.parse(text);
     })()
   `;
-    const result = await page.evaluate(js);
+    let result;
+    try {
+        result = await page.evaluate(js);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new CommandExecutionError(`Douyin API request failed: ${message}`);
+    }
+    if (result === null || result === undefined) {
+        throw new CommandExecutionError('Empty response from Douyin API');
+    }
     if (result && typeof result === 'object' && 'status_code' in result) {
         const code = result.status_code;
         if (code !== 0) {

package/clis/douyin/_shared/browser-fetch.test.js

@@ -27,4 +27,17 @@ describe('browserFetch', () => {
         const result = await browserFetch(page, 'GET', 'https://creator.douyin.com/api/test');
         expect(result).toEqual({ some_field: 'value' });
     });
+    it('throws on empty response body (null from evaluate)', async () => {
+        const page = makePage(null);
+        await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test')).rejects.toThrow('Empty response from Douyin API');
+    });
+    it('throws on undefined response body', async () => {
+        const page = makePage(undefined);
+        await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test')).rejects.toThrow('Empty response from Douyin API');
+    });
+    it('wraps browser-side fetch or JSON parse failures', async () => {
+        const page = makePage(null);
+        page.evaluate.mockRejectedValueOnce(new SyntaxError('Unexpected token < in JSON'));
+        await expect(browserFetch(page, 'GET', 'https://creator.douyin.com/api/test')).rejects.toThrow('Douyin API request failed: Unexpected token < in JSON');
+    });
 });

package/clis/douyin/stats.js

@@ -8,7 +8,7 @@ cli({
     domain: 'creator.douyin.com',
     strategy: Strategy.COOKIE,
     args: [
-        { name: 'aweme_id', required: true, positional: true },
+        { name: 'aweme_id', required: true, positional: true, help: '抖音作品 ID（aweme_id，可从作品 URL 末尾获取）' },
     ],
     columns: ['metric', 'value'],
     func: async (page, kwargs) => {

package/clis/douyin/update.js

@@ -10,7 +10,7 @@ cli({
     domain: 'creator.douyin.com',
     strategy: Strategy.COOKIE,
     args: [
-        { name: 'aweme_id', required: true, positional: true },
+        { name: 'aweme_id', required: true, positional: true, help: '抖音作品 ID（aweme_id，可从作品 URL 末尾获取）' },
         { name: 'reschedule', default: '', help: '新的发布时间（ISO8601 或 Unix 秒）' },
         { name: 'caption', default: '', help: '新的正文内容' },
     ],

package/clis/jike/search.js

@@ -15,7 +15,7 @@ cli({
     strategy: Strategy.COOKIE,
     browser: true,
     args: [
-        { name: 'query', type: 'string', required: true, positional: true },
+        { name: 'query', type: 'string', required: true, positional: true, help: '即刻搜索关键词' },
         { name: 'limit', type: 'int', default: 20 },
     ],
     columns: ['id', 'author', 'content', 'likes', 'comments', 'time', 'url'],

package/clis/reddit/search.js

@@ -8,7 +8,7 @@ cli({
     strategy: Strategy.COOKIE,
     browser: true,
     args: [
-        { name: 'query', type: 'string', required: true, positional: true },
+        { name: 'query', type: 'string', required: true, positional: true, help: 'Reddit search query' },
         {
             name: 'subreddit',
             type: 'string',

package/clis/reddit/subreddit.js

@@ -8,7 +8,7 @@ cli({
     strategy: Strategy.COOKIE,
     browser: true,
     args: [
-        { name: 'name', type: 'string', required: true, positional: true },
+        { name: 'name', type: 'string', required: true, positional: true, help: 'Subreddit name (no `r/` prefix; e.g. `python`)' },
         {
             name: 'sort',
             type: 'string',

package/clis/reddit/user-comments.js

@@ -8,7 +8,7 @@ cli({
     strategy: Strategy.COOKIE,
     browser: true,
     args: [
-        { name: 'username', type: 'string', required: true, positional: true },
+        { name: 'username', type: 'string', required: true, positional: true, help: 'Reddit username (no `u/` prefix needed)' },
         { name: 'limit', type: 'int', default: 15 },
     ],
     columns: ['subreddit', 'score', 'body', 'url'],

package/clis/reddit/user-posts.js

@@ -8,7 +8,7 @@ cli({
     strategy: Strategy.COOKIE,
     browser: true,
     args: [
-        { name: 'username', type: 'string', required: true, positional: true },
+        { name: 'username', type: 'string', required: true, positional: true, help: 'Reddit username (no `u/` prefix needed)' },
         { name: 'limit', type: 'int', default: 15 },
     ],
     columns: ['title', 'subreddit', 'score', 'comments', 'url'],

package/clis/reddit/user.js

@@ -8,7 +8,7 @@ cli({
     strategy: Strategy.COOKIE,
     browser: true,
     args: [
-        { name: 'username', type: 'string', required: true, positional: true },
+        { name: 'username', type: 'string', required: true, positional: true, help: 'Reddit username (no `u/` prefix needed)' },
     ],
     columns: ['field', 'value'],
     pipeline: [

package/clis/twitter/article.js

@@ -1,6 +1,7 @@
 import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
 import { cli, Strategy } from '@jackwener/opencli/registry';
 import { resolveTwitterQueryId } from './shared.js';
+import { TWITTER_BEARER_TOKEN } from './utils.js';
 const TWEET_RESULT_BY_REST_ID_QUERY_ID = '7xflPyRiUxGVbJd4uWmbfg';
 cli({
     site: 'twitter',
@@ -57,7 +58,7 @@ cli({
         const ct0 = document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1];
         if (!ct0) return {error: 'No ct0 cookie — not logged into x.com'};
 
-        const bearer = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
+        const bearer = ${JSON.stringify(TWITTER_BEARER_TOKEN)};
         const headers = {
           'Authorization': 'Bearer ' + decodeURIComponent(bearer),
           'X-Csrf-Token': ct0,