@cvtoolman/huawei-lts-cli 2.0.0 → 2.1.1

package/dist/lts/pagination.js

@@ -1,12 +1,31 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.cursorDirection = cursorDirection;
+exports.MAX_PAGE_SIZE = exports.CONTINUE_SEARCH_TYPE = void 0;
 exports.nextCursor = nextCursor;
+exports.shiftLineNum = shiftLineNum;
 exports.fetchAllPages = fetchAllPages;
-/** 倒序（新到旧）→ 往更早读；正序 → 往更晚读。 */
-function cursorDirection(reverse) {
-    return reverse ? 'backwards' : 'forwards';
-}
+/**
+ * LTS 的翻页语义：服务端没有 offset，只有「从某个 line_num 继续读」。
+ *
+ * 第一页不带游标；之后每页把上一页最后一条的 line_num 作为续读起点。
+ *
+ * 续读方向只有一种：`forwards`，它表示「沿当前排序方向继续」，与 `is_desc` 无关。
+ * 实测（2026-10，cn-south-1）降序查询配 `backwards` 时，第二页与第一页的 line_num
+ * 重叠 38%~100%：正序配 backwards 更直接，返回的就是第一页那一段。而且全程不报
+ * 错。所以这里不再按排序方向推方向——反了会静默产出重复数据，比报错难查得多。
+ * 对照：无论升降序，`forwards` 连翻 10 页、每页 100 条，边界重叠为 0。
+ *
+ * 游标是排他的：从 line_num=L 续读只会返回严格在 L 之后的记录。但 line_num 在
+ * 纳秒级并不唯一——同一纳秒的多条日志共用一个值，一组日志正好被页边界切开时，
+ * 后半组会被整组跳过（实测：单页查得到 18 条，--all 只拿到 17 条）。所以续读
+ * 游标取 L 的前一格（正序 L-1、降序 L+1），让下一页从这一组的组头开始读，再按
+ * 「line_num + 内容 + labels」跳过本页已经收走的那几条。line_num 不唯一，不能
+ * 只按它去重，否则会把同纳秒的真实日志一起丢掉。
+ */
+/** 续读游标时唯一正确的 search_type。 */
+exports.CONTINUE_SEARCH_TYPE = 'forwards';
+/** 服务端单页上限：limit=5001 直接回 LTS.1903 "limit must be between 1 and 5000"。 */
+exports.MAX_PAGE_SIZE = 5000;
 /** 游标取本页最后一条的 line_num；拿不到就返回 undefined（调用方据此停止）。 */
 function nextCursor(logs) {
     if (logs.length === 0)
@@ -14,39 +33,119 @@ function nextCursor(logs) {
     const last = logs[logs.length - 1].lineNum;
     return last ? last : undefined;
 }
+/** line_num 是纳秒整数；游标前后挪一格，才能把被页边界切开的同值组重新纳入。 */
+function shiftLineNum(lineNum, delta) {
+    if (!/^\d+$/.test(lineNum))
+        return undefined;
+    return (BigInt(lineNum) + delta).toString();
+}
+/** line_num 不唯一，同纳秒同内容也可能是两条真实日志；回放判定要带上内容与 labels。 */
+function entrySignature(entry) {
+    const labels = Object.keys(entry.labels)
+        .sort()
+        .map((key) => `${key}=${entry.labels[key]}`)
+        .join('\u0001');
+    return `${entry.content}\u0000${labels}`;
+}
+/** 统计某一 line_num 组内各签名的条数，下一页据此跳过回放。 */
+function countSignatures(entries, lineNum) {
+    const counts = new Map();
+    for (const entry of entries) {
+        if (entry.lineNum !== lineNum)
+            continue;
+        const signature = entrySignature(entry);
+        counts.set(signature, (counts.get(signature) ?? 0) + 1);
+    }
+    return counts;
+}
+function mergeCounts(into, extra) {
+    for (const [signature, count] of extra) {
+        into.set(signature, (into.get(signature) ?? 0) + count);
+    }
+    return into;
+}
 async function fetchAllPages(fetchPage, params, options = {}) {
     const max = options.max ?? 10000;
-    const pageSize = options.pageSize ?? params.limit ?? 100;
-    const direction = cursorDirection(params.reverse);
+    // 每页条数不能超过服务端上限，否则整次 --all 直接报错
+    const basePageSize = Math.min(options.pageSize ?? params.limit ?? 100, exports.MAX_PAGE_SIZE);
+    // forwards 沿当前排序方向推进：正序越来越晚（-1），降序越来越早（+1）
+    const step = params.reverse === true ? 1n : -1n;
     const collected = [];
     let cursor;
+    // 上一页末条所在的 line_num 组；下一页从该组组头重读，组内已收走的条目按签名跳过
+    let boundaryLineNum;
+    let boundaryCounts = new Map();
     let isQueryComplete;
     let truncated = false;
     while (collected.length < max) {
-        const page = await fetchPage({
-            ...params,
-            limit: pageSize,
-            lineNum: cursor,
-            searchType: cursor ? direction : undefined,
-        });
-        if (page.isQueryComplete === false) {
-            isQueryComplete = false;
+        // 重读组头会带回已收走的同值组；多要这么多条，保证每页仍有 pageSize 条新数据
+        const boundarySize = [...boundaryCounts.values()].reduce((sum, count) => sum + count, 0);
+        let pageSize = Math.min(basePageSize + boundarySize, exports.MAX_PAGE_SIZE);
+        let pageLength = 0;
+        let fresh = [];
+        let reachedEnd = false;
+        // 同纳秒的日志多到一页装不下时，重读组头永远只能拿到前 pageSize 条；
+        // 放大一页才可能越过这一组。
+        while (true) {
+            const page = await fetchPage({
+                ...params,
+                limit: pageSize,
+                lineNum: cursor,
+                searchType: cursor ? exports.CONTINUE_SEARCH_TYPE : undefined,
+            });
+            if (page.isQueryComplete === false) {
+                isQueryComplete = false;
+            }
+            else if (isQueryComplete === undefined && page.isQueryComplete === true) {
+                isQueryComplete = true;
+            }
+            pageLength = page.logs.length;
+            if (pageLength === 0) {
+                reachedEnd = true;
+                break;
+            }
+            const pending = new Map(boundaryCounts);
+            fresh = [];
+            for (const entry of page.logs) {
+                if (boundaryLineNum !== undefined && entry.lineNum === boundaryLineNum) {
+                    const signature = entrySignature(entry);
+                    const remaining = pending.get(signature) ?? 0;
+                    if (remaining > 0) {
+                        pending.set(signature, remaining - 1);
+                        continue;
+                    }
+                }
+                fresh.push(entry);
+            }
+            if (fresh.length > 0)
+                break;
+            // 不满一页且全是回放 = 同值组之后再无日志，已经到底
+            if (pageLength < pageSize) {
+                reachedEnd = true;
+                break;
+            }
+            if (pageSize >= exports.MAX_PAGE_SIZE) {
+                truncated = true;
+                reachedEnd = true;
+                break;
+            }
+            pageSize = Math.min(pageSize * 2, exports.MAX_PAGE_SIZE);
         }
-        else if (isQueryComplete === undefined && page.isQueryComplete === true) {
-            isQueryComplete = true;
-        }
-        if (page.logs.length === 0)
-            break;
-        const next = nextCursor(page.logs);
-        // 服务端把同一段原样又给了一遍：整页丢弃并停止。
-        // 先判游标再入账，否则重放的一页会被当成新数据拼进结果。
-        if (cursor !== undefined && next === cursor)
+        if (reachedEnd)
             break;
-        collected.push(...page.logs);
-        // 不满一页 = 没有更多了；拿不到游标则无法继续翻页
-        if (page.logs.length < pageSize || !next)
+        collected.push(...fresh);
+        const lastLineNum = fresh[fresh.length - 1]?.lineNum;
+        // 不满一页 = 没有更多了；游标推不动则无法继续翻页
+        const next = lastLineNum ? shiftLineNum(lastLineNum, step) : undefined;
+        if (pageLength < pageSize || !next)
             break;
         cursor = next;
+        // 同值组可能横跨多页：计数必须累计，否则重读组头时会把前几页收走的条目再收一遍
+        boundaryCounts =
+            lastLineNum === boundaryLineNum
+                ? mergeCounts(boundaryCounts, countSignatures(fresh, lastLineNum))
+                : countSignatures(fresh, lastLineNum);
+        boundaryLineNum = lastLineNum;
         if (collected.length >= max) {
             truncated = true;
         }

package/package.json

@@ -1,57 +1,57 @@
-{
-  "name": "@cvtoolman/huawei-lts-cli",
-  "version": "2.0.0",
-  "description": "Command-line tool for querying Huawei Cloud Log Tank Service (LTS) logs",
-  "main": "dist/cli/index.js",
-  "bin": {
-    "lts-cli": "./dist/cli/index.js"
-  },
-  "files": [
-    "dist/"
-  ],
-  "scripts": {
-    "build": "tsc",
-    "dev": "tsx src/cli/index.ts",
-    "start": "node dist/cli/index.js",
-    "test": "node --import tsx --test \"test/**/*.test.ts\"",
-    "typecheck": "tsc --noEmit",
-    "clean": "node -e \"require('fs').rmSync('dist', {recursive: true, force: true})\"",
-    "prepublishOnly": "npm run clean && npm run build"
-  },
-  "keywords": [
-    "huawei-cloud",
-    "huawei",
-    "lts",
-    "log-tank-service",
-    "logs",
-    "cli",
-    "log-query",
-    "cloud-logging"
-  ],
-  "author": "cvtoolman",
-  "license": "MIT",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/jcxy/huawei-lts-cli.git"
-  },
-  "bugs": {
-    "url": "https://github.com/jcxy/huawei-lts-cli/issues"
-  },
-  "homepage": "https://github.com/jcxy/huawei-lts-cli#readme",
-  "engines": {
-    "node": ">=18.0.0"
-  },
-  "dependencies": {
-    "@huaweicloud/huaweicloud-sdk-core": "^3.1.200",
-    "@huaweicloud/huaweicloud-sdk-lts": "^3.1.200",
-    "chalk": "^4.1.2",
-    "cli-table3": "^0.6.5",
-    "commander": "^15.0.0",
-    "uuid": "^9.0.0"
-  },
-  "devDependencies": {
-    "@types/node": "^26.0.0",
-    "tsx": "^4.22.4",
-    "typescript": "^6.0.3"
-  }
-}
+{
+  "name": "@cvtoolman/huawei-lts-cli",
+  "version": "2.1.1",
+  "description": "Command-line tool for querying Huawei Cloud Log Tank Service (LTS) logs",
+  "main": "dist/cli/index.js",
+  "bin": {
+    "lts-cli": "dist/cli/index.js"
+  },
+  "files": [
+    "dist/"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/cli/index.ts",
+    "start": "node dist/cli/index.js",
+    "test": "node --import tsx --test \"test/**/*.test.ts\"",
+    "typecheck": "tsc --noEmit",
+    "clean": "node -e \"require('fs').rmSync('dist', {recursive: true, force: true})\"",
+    "prepublishOnly": "npm run clean && npm run build"
+  },
+  "keywords": [
+    "huawei-cloud",
+    "huawei",
+    "lts",
+    "log-tank-service",
+    "logs",
+    "cli",
+    "log-query",
+    "cloud-logging"
+  ],
+  "author": "cvtoolman",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jcxy/huawei-lts-cli.git"
+  },
+  "bugs": {
+    "url": "https://github.com/jcxy/huawei-lts-cli/issues"
+  },
+  "homepage": "https://github.com/jcxy/huawei-lts-cli#readme",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@huaweicloud/huaweicloud-sdk-core": "^3.1.200",
+    "@huaweicloud/huaweicloud-sdk-lts": "^3.1.200",
+    "chalk": "^4.1.2",
+    "cli-table3": "^0.6.5",
+    "commander": "^15.0.0",
+    "uuid": "^9.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^26.0.0",
+    "tsx": "^4.22.4",
+    "typescript": "^6.0.3"
+  }
+}

package/dist/cli/options.js

@@ -1,15 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.queryOptions = void 0;
-exports.queryOptions = {
-    groupId: { short: '-g', long: '--group-id <groupId>', description: 'Log group ID', required: true },
-    streamId: { short: '-s', long: '--stream-id <streamId>', description: 'Log stream ID', required: true },
-    startTime: { short: '--start-time <startTime>', long: '--start-time <startTime>', description: 'Start time (ISO 8601)', required: true },
-    endTime: { short: '--end-time <endTime>', long: '--end-time <endTime>', description: 'End time (ISO 8601)', required: true },
-    keyword: { short: '-k', long: '--keyword <keyword>', description: 'Search keyword', required: false },
-    limit: { short: '-l', long: '--limit <limit>', description: 'Limit number of results', required: false },
-    offset: { short: '-o', long: '--offset <offset>', description: 'Offset for pagination', required: false },
-    reverse: { short: '-r', long: '--reverse', description: 'Reverse order', required: false },
-    format: { short: '-f', long: '--format <format>', description: 'Output format: json, table, pretty', required: false },
-    noPaginate: { short: '', long: '--no-paginate', description: 'Disable pagination', required: false },
-};

package/dist/config/profile.js

@@ -1,108 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getProfilePath = getProfilePath;
-exports.profileExists = profileExists;
-exports.writeProfileSkeleton = writeProfileSkeleton;
-const fs = __importStar(require("fs"));
-const path = __importStar(require("path"));
-const store_1 = require("./store");
-/**
- * `profile.md` 是给人写、给 AI 读的业务词表。CLI 从不读它。
- *
- * 分界线：`config.json` 存 CLI 的行为（凭证、traceField、默认 group/stream）；
- * `profile.md` 存 API 拿不到的东西（业务词 → 日志里的值、命名约定、黑话）。
- * 字段清单、日志组/流 ID 一律不进 profile —— 那是第二份会过期的真相，
- * 用 `lts-cli groups` / `streams` / `fields` 现查。
- */
-function getProfilePath() {
-    return path.join((0, store_1.getConfigDir)(), 'profile.md');
-}
-function profileExists() {
-    return fs.existsSync(getProfilePath());
-}
-const SKELETON = `# LTS 业务词表（本机私有）
-
-> 这份文件给人改、给 AI 读。\`lts-cli\` 本身**不读**它——CLI 的行为配置在 \`~/.lts-cli/config.json\`。
->
-> 只写 API 拿不到的东西：业务词怎么落到日志里的值、团队自己的命名约定、黑话。
-> **不要**把日志组/日志流 ID、字段清单写进来——用 \`lts-cli groups\` / \`lts-cli streams\` /
-> \`lts-cli fields\` 现查。抄进文档的那一刻它就开始过期。
-
-## 1. 业务称呼 → 日志里的取值
-
-查日志前先在这里翻译，避免每次重新猜应用名。
-
-| 业务称呼 | 日志中的取值 | 说明 |
-| --- | --- | --- |
-| 订单服务 | （填字段值和值） | 例如 appName 或 service_name 的取值 |
-
-## 2. 字段含义
-
-只写「这个字段在我们这里表示什么」。类型、是否大小写敏感、分词符请用 \`lts-cli fields\` 查。
-
-| 字段 | 含义 |
-| --- | --- |
-| （字段名） | （说明） |
-
-## 3. 命名约定与分词陷阱
-
-- 哪些值含连字符/下划线，必须加双引号才会被当成一个词
-- 值的大小写约定（全大写？小写？）
-- 哪些字段看起来该有索引、实际上没建（没建索引的字段只能全文检索命中）
-
-## 4. 团队黑话
-
-| 说法 | 实际指什么 |
-| --- | --- |
-| （黑话） | （含义） |
-
-## 5. 常用排查入口
-
-| 想查什么 | 建议命令 |
-| --- | --- |
-| 某个 ID 的完整链路 | \`lts-cli trace <id>\` |
-| 某业务最近的报错 | \`lts-cli query --last 1h -q '(字段):(值)'\` |
-| 先看清楚这条流有什么字段 | \`lts-cli fields\` |
-`;
-function writeProfileSkeleton() {
-    const target = getProfilePath();
-    if (fs.existsSync(target)) {
-        return { path: target, created: false };
-    }
-    fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
-    fs.writeFileSync(target, SKELETON, { encoding: 'utf-8', mode: 0o600 });
-    return { path: target, created: true };
-}