gangtise-openapi-cli 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +97 -4
- package/dist/src/cli.js +142 -29
- package/dist/src/core/asyncContent.js +13 -4
- package/dist/src/core/client.js +208 -125
- package/dist/src/core/download.js +4 -0
- package/dist/src/core/endpoints.js +49 -3
- package/dist/src/core/output.js +64 -0
- package/dist/src/core/printer.js +7 -3
- package/dist/src/core/quoteSharding.js +81 -0
- package/dist/src/core/titleCache.js +58 -10
- package/dist/src/core/transport.js +91 -0
- package/dist/src/version.js +1 -1
- package/package.json +1 -1
|
@@ -4,24 +4,65 @@ import path from "node:path";
|
|
|
4
4
|
export const DEFAULT_TITLE_CACHE_PATH = path.join(os.homedir(), ".config", "gangtise", "title-cache.json");
|
|
5
5
|
export const TITLE_LOOKUP_SIZE = 200;
|
|
6
6
|
const TITLE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
|
7
|
-
|
|
7
|
+
// Per-process in-memory snapshot of the cache. We read the file at most once,
|
|
8
|
+
// merge subsequent writes in memory, and flush atomically. This avoids the
|
|
9
|
+
// "read whole file → modify → write whole file" pattern firing on every list
|
|
10
|
+
// command (which got expensive once dozens of endpoints accumulated).
|
|
11
|
+
let memoryCache = null;
|
|
12
|
+
let memoryCachePath = null;
|
|
13
|
+
let pendingWrite = null;
|
|
14
|
+
let dirty = false;
|
|
15
|
+
async function loadInto(filePath) {
|
|
16
|
+
if (memoryCache && memoryCachePath === filePath)
|
|
17
|
+
return memoryCache;
|
|
8
18
|
try {
|
|
9
19
|
const content = await fs.readFile(filePath, "utf8");
|
|
10
20
|
const parsed = JSON.parse(content);
|
|
11
|
-
|
|
12
|
-
return parsed;
|
|
13
|
-
}
|
|
21
|
+
memoryCache = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
|
14
22
|
}
|
|
15
23
|
catch {
|
|
16
|
-
|
|
24
|
+
memoryCache = {};
|
|
17
25
|
}
|
|
18
|
-
|
|
26
|
+
memoryCachePath = filePath;
|
|
27
|
+
return memoryCache;
|
|
19
28
|
}
|
|
20
|
-
export async function
|
|
21
|
-
|
|
22
|
-
|
|
29
|
+
export async function readTitleCache(filePath = DEFAULT_TITLE_CACHE_PATH) {
|
|
30
|
+
return loadInto(filePath);
|
|
31
|
+
}
|
|
32
|
+
async function flush(filePath) {
|
|
33
|
+
if (!dirty || !memoryCache)
|
|
34
|
+
return;
|
|
35
|
+
dirty = false;
|
|
36
|
+
const snapshot = JSON.stringify(memoryCache);
|
|
23
37
|
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
24
|
-
|
|
38
|
+
// Atomic-ish: write to temp file then rename (rename is atomic within a fs).
|
|
39
|
+
const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}`;
|
|
40
|
+
await fs.writeFile(tmp, snapshot, { encoding: "utf8", mode: 0o600 });
|
|
41
|
+
try {
|
|
42
|
+
await fs.rename(tmp, filePath);
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
await fs.unlink(tmp).catch(() => { });
|
|
46
|
+
throw error;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
export async function writeTitleCache(endpoint, titles, filePath = DEFAULT_TITLE_CACHE_PATH) {
|
|
50
|
+
const data = await loadInto(filePath);
|
|
51
|
+
const existing = data[endpoint]?.titles ?? {};
|
|
52
|
+
data[endpoint] = { titles: { ...existing, ...titles }, ts: Date.now() };
|
|
53
|
+
dirty = true;
|
|
54
|
+
// Coalesce concurrent writes: the in-flight flush picks up everything dirty.
|
|
55
|
+
if (pendingWrite)
|
|
56
|
+
return pendingWrite;
|
|
57
|
+
pendingWrite = (async () => {
|
|
58
|
+
try {
|
|
59
|
+
await flush(filePath);
|
|
60
|
+
}
|
|
61
|
+
finally {
|
|
62
|
+
pendingWrite = null;
|
|
63
|
+
}
|
|
64
|
+
})();
|
|
65
|
+
return pendingWrite;
|
|
25
66
|
}
|
|
26
67
|
export function lookupTitleCache(data, endpoint, id) {
|
|
27
68
|
const entry = data[endpoint];
|
|
@@ -44,3 +85,10 @@ export function extractTitles(items, cache) {
|
|
|
44
85
|
}
|
|
45
86
|
return titles;
|
|
46
87
|
}
|
|
88
|
+
/** Test-only hook to reset the in-memory snapshot between cases. */
|
|
89
|
+
export function __resetTitleCacheForTests() {
|
|
90
|
+
memoryCache = null;
|
|
91
|
+
memoryCachePath = null;
|
|
92
|
+
pendingWrite = null;
|
|
93
|
+
dirty = false;
|
|
94
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { Agent } from "undici";
|
|
2
|
+
import { ApiError } from "./errors.js";
|
|
3
|
+
let cachedDispatcher = null;
|
|
4
|
+
export function getDispatcher() {
|
|
5
|
+
if (!cachedDispatcher) {
|
|
6
|
+
cachedDispatcher = new Agent({
|
|
7
|
+
keepAliveTimeout: 60_000,
|
|
8
|
+
keepAliveMaxTimeout: 600_000,
|
|
9
|
+
connections: 16,
|
|
10
|
+
pipelining: 1,
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
return cachedDispatcher;
|
|
14
|
+
}
|
|
15
|
+
export async function runWithConcurrency(items, concurrency, fn) {
|
|
16
|
+
if (items.length === 0)
|
|
17
|
+
return [];
|
|
18
|
+
const limit = Math.max(1, Math.min(concurrency, items.length));
|
|
19
|
+
const results = new Array(items.length);
|
|
20
|
+
let next = 0;
|
|
21
|
+
async function worker() {
|
|
22
|
+
while (true) {
|
|
23
|
+
const index = next++;
|
|
24
|
+
if (index >= items.length)
|
|
25
|
+
return;
|
|
26
|
+
results[index] = await fn(items[index], index);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
await Promise.all(Array.from({ length: limit }, () => worker()));
|
|
30
|
+
return results;
|
|
31
|
+
}
|
|
32
|
+
const RETRYABLE_HTTP_STATUS = new Set([429, 500, 502, 503, 504]);
|
|
33
|
+
const RETRYABLE_NETWORK_CODES = new Set(["ECONNRESET", "ETIMEDOUT", "ENOTFOUND", "EAI_AGAIN", "UND_ERR_SOCKET", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT"]);
|
|
34
|
+
const RETRYABLE_API_CODES = new Set(["999999"]);
|
|
35
|
+
function isRetryableError(error) {
|
|
36
|
+
if (error && typeof error === "object" && error.__retryable === true) {
|
|
37
|
+
return true;
|
|
38
|
+
}
|
|
39
|
+
if (error instanceof ApiError) {
|
|
40
|
+
if (error.statusCode != null && RETRYABLE_HTTP_STATUS.has(error.statusCode))
|
|
41
|
+
return true;
|
|
42
|
+
if (error.code && RETRYABLE_API_CODES.has(error.code))
|
|
43
|
+
return true;
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
if (error && typeof error === "object" && "code" in error) {
|
|
47
|
+
const code = String(error.code);
|
|
48
|
+
if (RETRYABLE_NETWORK_CODES.has(code))
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
if (error instanceof Error && /timeout|ETIMEDOUT|ECONNRESET|socket hang up/i.test(error.message)) {
|
|
52
|
+
return true;
|
|
53
|
+
}
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
export function markRetryable(error) {
|
|
57
|
+
return Object.assign(error, { __retryable: true });
|
|
58
|
+
}
|
|
59
|
+
export async function withRetry(fn, options = {}) {
|
|
60
|
+
const retries = options.retries ?? 2;
|
|
61
|
+
const baseDelay = options.baseDelayMs ?? 400;
|
|
62
|
+
const maxDelay = options.maxDelayMs ?? 4_000;
|
|
63
|
+
let attempt = 0;
|
|
64
|
+
while (true) {
|
|
65
|
+
try {
|
|
66
|
+
return await fn();
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
if (attempt >= retries || !isRetryableError(error))
|
|
70
|
+
throw error;
|
|
71
|
+
const jitter = Math.random() * baseDelay;
|
|
72
|
+
const delay = Math.min(maxDelay, baseDelay * 2 ** attempt + jitter);
|
|
73
|
+
options.onRetry?.(attempt + 1, error, delay);
|
|
74
|
+
await new Promise(resolve => setTimeout(resolve, delay));
|
|
75
|
+
attempt++;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
let verboseEnabled = process.env.GANGTISE_VERBOSE === "1" || process.env.GANGTISE_VERBOSE === "true";
|
|
80
|
+
export function setVerbose(value) {
|
|
81
|
+
verboseEnabled = value;
|
|
82
|
+
}
|
|
83
|
+
export function isVerbose() {
|
|
84
|
+
return verboseEnabled;
|
|
85
|
+
}
|
|
86
|
+
export function logTiming(label, durationMs, extra) {
|
|
87
|
+
if (!verboseEnabled)
|
|
88
|
+
return;
|
|
89
|
+
const ms = durationMs.toFixed(0).padStart(5, " ");
|
|
90
|
+
process.stderr.write(`[gangtise] ${ms}ms ${label}${extra ? ` (${extra})` : ""}\n`);
|
|
91
|
+
}
|
package/dist/src/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
// Auto-generated — DO NOT EDIT
|
|
2
|
-
export const CLI_VERSION = "0.
|
|
2
|
+
export const CLI_VERSION = "0.12.0";
|