@dev-pi2pie/word-counter 0.1.4-canary.1 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -5
- package/dist/esm/bin.mjs +172 -287
- package/dist/esm/bin.mjs.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -151,8 +151,10 @@ word-counter --path ./examples/test-case-multi-files-support --jobs 4
|
|
|
151
151
|
Quick policy:
|
|
152
152
|
|
|
153
153
|
- no `--jobs` and `--jobs 1` are equivalent baseline behavior.
|
|
154
|
-
- `--jobs
|
|
154
|
+
- `--jobs 1`: async main-thread `load+count` baseline.
|
|
155
|
+
- `--jobs > 1`: worker `load+count` with async fallback when workers are unavailable.
|
|
155
156
|
- if requested `--jobs` exceeds host `suggestedMaxJobs` (from `--print-jobs-limit`), the CLI warns and runs with the suggested limit as a safety cap.
|
|
157
|
+
- use `--quiet-warnings` to suppress non-fatal warning lines (for example jobs-limit advisory and worker-fallback warning).
|
|
156
158
|
|
|
157
159
|
Inspect host jobs diagnostics:
|
|
158
160
|
|
|
@@ -160,6 +162,8 @@ Inspect host jobs diagnostics:
|
|
|
160
162
|
word-counter --print-jobs-limit
|
|
161
163
|
```
|
|
162
164
|
|
|
165
|
+
`--print-jobs-limit` must be used alone (no other inputs or runtime flags).
|
|
166
|
+
|
|
163
167
|
For full policy details, JSON parity expectations (`--misc`, `--total-of whitespace,words`), and benchmark standards, see [`docs/batch-jobs-usage-guide.md`](docs/batch-jobs-usage-guide.md).
|
|
164
168
|
|
|
165
169
|
### Stable Path Resolution Contract
|
|
@@ -220,6 +224,8 @@ For additional usage details and troubleshooting, see [`docs/regex-usage-guide.m
|
|
|
220
224
|
|
|
221
225
|
### Debugging Diagnostics (`--debug`)
|
|
222
226
|
|
|
227
|
+
Noise policy: default output shows errors + warnings; `--debug` enables diagnostics; `--verbose` enables per-item diagnostics; `--quiet-warnings` suppresses warnings.
|
|
228
|
+
|
|
223
229
|
`--debug` remains the diagnostics gate and now defaults to `compact` event volume:
|
|
224
230
|
|
|
225
231
|
- lifecycle/stage timing events
|
|
@@ -252,7 +258,7 @@ word-counter --path ./examples/test-case-multi-files-support --debug --debug-rep
|
|
|
252
258
|
word-counter --path ./examples/test-case-multi-files-support --debug --debug-report ./logs/debug.jsonl --debug-tee
|
|
253
259
|
```
|
|
254
260
|
|
|
255
|
-
Skip details stay debug-gated and can
|
|
261
|
+
Skip details stay debug-gated and can be suppressed with `--quiet-skips`.
|
|
256
262
|
|
|
257
263
|
## How It Works
|
|
258
264
|
|
|
@@ -491,8 +497,9 @@ Example (trimmed):
|
|
|
491
497
|
"frontmatterType": "yaml",
|
|
492
498
|
"total": 7,
|
|
493
499
|
"items": [
|
|
494
|
-
{ "name": "content", "source": "frontmatter", "result": { "total":
|
|
495
|
-
{ "name": "content", "source": "
|
|
500
|
+
{ "name": "content", "source": "frontmatter", "result": { "total": 4 } },
|
|
501
|
+
{ "name": "content", "source": "frontmatter", "result": { "total": 2 } },
|
|
502
|
+
{ "name": "content", "source": "content", "result": { "total": 5 } }
|
|
496
503
|
]
|
|
497
504
|
}
|
|
498
505
|
```
|
|
@@ -585,7 +592,7 @@ word-counter --include-whitespace "Hi\tthere\n"
|
|
|
585
592
|
word-counter --misc "Hi\tthere\n"
|
|
586
593
|
```
|
|
587
594
|
|
|
588
|
-
In the CLI, `--include-whitespace` implies
|
|
595
|
+
In the CLI, `--include-whitespace` implies `--non-words` (same behavior as `--misc`). `--non-words` alone does not include whitespace. When enabled, whitespace counts appear under `nonWords.whitespace`, and `total = words + nonWords` (emoji + symbols + punctuation + whitespace). JSON output also includes top-level `counts` when `nonWords` is enabled. See `docs/schemas/whitespace-categories.md` for how whitespace is categorized.
|
|
589
596
|
|
|
590
597
|
Example JSON (trimmed):
|
|
591
598
|
|
package/dist/esm/bin.mjs
CHANGED
|
@@ -5,8 +5,8 @@ import { closeSync, createWriteStream, existsSync, mkdirSync, openSync, readFile
|
|
|
5
5
|
import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import os from "node:os";
|
|
8
|
-
import { readFile, readdir, stat } from "node:fs/promises";
|
|
9
8
|
import { parseDocument } from "yaml";
|
|
9
|
+
import { readFile, readdir, stat } from "node:fs/promises";
|
|
10
10
|
|
|
11
11
|
//#region \0rolldown/runtime.js
|
|
12
12
|
var __create = Object.create;
|
|
@@ -353,7 +353,7 @@ function parseJobsOption(value) {
|
|
|
353
353
|
return parsed;
|
|
354
354
|
}
|
|
355
355
|
function configureProgramOptions(program, parseMode) {
|
|
356
|
-
program.addOption(new Option("-m, --mode <mode>", "breakdown mode").choices(MODE_CHOICES).argParser(parseMode).default("chunk")).addOption(new Option("-f, --format <format>", "output format").choices(FORMAT_CHOICES).default("standard")).addOption(new Option("--section <section>", "document section mode").choices(SECTION_CHOICES).default("all")).addOption(new Option("--path-mode <mode>", "path resolution mode: auto (default) expands directories; manual treats --path values as literal files").choices(PATH_MODE_CHOICES).default("auto")).option("--latin-language <language>", "hint a language tag for Latin script text").option("--latin-tag <tag>", "hint a BCP 47 tag for Latin script text").option("--latin-locale <locale>", "legacy alias of --latin-language").option("--latin-hint <tag>=<pattern>", "add a custom Latin hint rule (repeatable)", collectLatinHintValue, []).option("--latin-hints-file <path>", "load custom Latin hint rules from a JSON file").option("--no-default-latin-hints", "disable built-in Latin hint rules").option("--han-language <language>", "hint a language tag for Han script text").option("--han-tag <tag>", "hint a BCP 47 tag for Han script text").option("--non-words", "collect emoji, symbols, and punctuation (excludes whitespace)").option("--include-whitespace", "include whitespace counts (implies
|
|
356
|
+
program.addOption(new Option("-m, --mode <mode>", "breakdown mode").choices(MODE_CHOICES).argParser(parseMode).default("chunk")).addOption(new Option("-f, --format <format>", "output format").choices(FORMAT_CHOICES).default("standard")).addOption(new Option("--section <section>", "document section mode").choices(SECTION_CHOICES).default("all")).addOption(new Option("--path-mode <mode>", "path resolution mode: auto (default) expands directories; manual treats --path values as literal files").choices(PATH_MODE_CHOICES).default("auto")).option("--latin-language <language>", "hint a language tag for Latin script text").option("--latin-tag <tag>", "hint a BCP 47 tag for Latin script text").option("--latin-locale <locale>", "legacy alias of --latin-language").option("--latin-hint <tag>=<pattern>", "add a custom Latin hint rule (repeatable)", collectLatinHintValue, []).option("--latin-hints-file <path>", "load custom Latin hint rules from a JSON file").option("--no-default-latin-hints", "disable built-in Latin hint rules").option("--han-language <language>", "hint a language tag for Han script text").option("--han-tag <tag>", "hint a BCP 47 tag for Han script text").option("--non-words", "collect emoji, symbols, and punctuation (excludes whitespace)").option("--include-whitespace", "include whitespace counts (implies --non-words; same as --misc)").option("--misc", "collect non-words plus whitespace (alias for --include-whitespace)").option("--total-of <parts>", "override total composition (comma-separated): words,emoji,symbols,punctuation,whitespace", parseTotalOfOption).option("--pretty", "pretty print JSON output", false).option("--debug", "enable debug diagnostics on stderr").option("--verbose", "emit verbose per-file debug diagnostics (requires --debug)").option("--debug-report [path]", "write debug diagnostics to a report file").option("--debug-report-tee", "mirror debug diagnostics to both report file and stderr").option("--debug-tee", "alias of --debug-report-tee").option("--merged", "show merged aggregate output (default)").option("--per-file", "show per-file output plus merged summary").option("--jobs <n>", "batch jobs in --path mode (1=async main-thread, >1=worker load+count)", parseJobsOption, 1).option("--print-jobs-limit", "print host jobs-limit JSON and exit (must be used alone)").option("--no-progress", "disable batch progress indicator").option("--keep-progress", "keep final batch progress line visible in standard mode").option("--no-recursive", "disable recursive directory traversal").option("--quiet-warnings", "suppress non-fatal warning diagnostics").option("--quiet-skips", "suppress debug skip output and per-file json skipped field").option("--include-ext <exts>", "comma-separated extensions to include during directory scanning", collectExtensionOption, []).option("--exclude-ext <exts>", "comma-separated extensions to exclude during directory scanning", collectExtensionOption, []).option("--regex <pattern>", "regex filter for directory-scanned paths (applies to --path directories only)").option("-p, --path <path>", "read input from file or directory (directories expand in auto mode by default)", collectPathValue, []).argument("[text...]", "text to count").showHelpAfterError();
|
|
357
357
|
}
|
|
358
358
|
|
|
359
359
|
//#endregion
|
|
@@ -428,7 +428,7 @@ var require_picocolors = /* @__PURE__ */ __commonJSMin(((exports, module) => {
|
|
|
428
428
|
//#endregion
|
|
429
429
|
//#region src/cli/program/version-embedded.ts
|
|
430
430
|
var import_picocolors = /* @__PURE__ */ __toESM(require_picocolors(), 1);
|
|
431
|
-
const EMBEDDED_PACKAGE_VERSION = "0.1.4
|
|
431
|
+
const EMBEDDED_PACKAGE_VERSION = "0.1.4";
|
|
432
432
|
|
|
433
433
|
//#endregion
|
|
434
434
|
//#region src/cli/program/version.ts
|
|
@@ -1958,26 +1958,6 @@ function compactCollectorSegmentsInCountResult(result) {
|
|
|
1958
1958
|
}
|
|
1959
1959
|
stripCollectorSegmentsFromWordCounterResult(result);
|
|
1960
1960
|
}
|
|
1961
|
-
async function buildBatchSummary(inputs, section, wcOptions, options = {}) {
|
|
1962
|
-
const preserveCollectorSegments = options.preserveCollectorSegments ?? true;
|
|
1963
|
-
const files = [];
|
|
1964
|
-
for (const input of inputs) {
|
|
1965
|
-
const result = section === "all" ? wc_default(input.content, wcOptions) : countSections(input.content, section, wcOptions);
|
|
1966
|
-
if (!preserveCollectorSegments) compactCollectorSegmentsInCountResult(result);
|
|
1967
|
-
files.push({
|
|
1968
|
-
path: input.path,
|
|
1969
|
-
result
|
|
1970
|
-
});
|
|
1971
|
-
options.onFileCounted?.({
|
|
1972
|
-
completed: files.length,
|
|
1973
|
-
total: inputs.length
|
|
1974
|
-
});
|
|
1975
|
-
}
|
|
1976
|
-
return finalizeBatchSummaryFromFileResults(files, section, wcOptions, {
|
|
1977
|
-
onFinalizeStart: options.onFinalizeStart,
|
|
1978
|
-
preserveCollectorSegments: options.preserveCollectorSegments
|
|
1979
|
-
});
|
|
1980
|
-
}
|
|
1981
1961
|
function finalizeBatchSummaryFromFileResults(files, section, wcOptions, options = {}) {
|
|
1982
1962
|
const preserveCollectorSegments = options.preserveCollectorSegments ?? true;
|
|
1983
1963
|
if (!preserveCollectorSegments) for (const file of files) compactCollectorSegmentsInCountResult(file.result);
|
|
@@ -1999,6 +1979,26 @@ function finalizeBatchSummaryFromFileResults(files, section, wcOptions, options
|
|
|
1999
1979
|
};
|
|
2000
1980
|
}
|
|
2001
1981
|
|
|
1982
|
+
//#endregion
|
|
1983
|
+
//#region src/cli/batch/jobs/queue.ts
|
|
1984
|
+
async function runBoundedQueue(total, requestedJobs, worker) {
|
|
1985
|
+
if (total === 0) return [];
|
|
1986
|
+
const safeRequestedJobs = Number.isFinite(requestedJobs) ? Math.floor(requestedJobs) : 1;
|
|
1987
|
+
const concurrency = Math.max(1, Math.min(total, safeRequestedJobs));
|
|
1988
|
+
const results = new Array(total);
|
|
1989
|
+
let nextIndex = 0;
|
|
1990
|
+
const runWorker = async () => {
|
|
1991
|
+
while (true) {
|
|
1992
|
+
const current = nextIndex;
|
|
1993
|
+
nextIndex += 1;
|
|
1994
|
+
if (current >= total) return;
|
|
1995
|
+
results[current] = await worker(current);
|
|
1996
|
+
}
|
|
1997
|
+
};
|
|
1998
|
+
await Promise.all(Array.from({ length: concurrency }, () => runWorker()));
|
|
1999
|
+
return results;
|
|
2000
|
+
}
|
|
2001
|
+
|
|
2002
2002
|
//#endregion
|
|
2003
2003
|
//#region src/cli/path/load.ts
|
|
2004
2004
|
function isProbablyBinary(buffer) {
|
|
@@ -2017,67 +2017,48 @@ function isProbablyBinary(buffer) {
|
|
|
2017
2017
|
}
|
|
2018
2018
|
|
|
2019
2019
|
//#endregion
|
|
2020
|
-
//#region src/cli/batch/jobs/
|
|
2021
|
-
async function
|
|
2022
|
-
if (
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2020
|
+
//#region src/cli/batch/jobs/read-input.ts
|
|
2021
|
+
async function readBatchInput(path, options) {
|
|
2022
|
+
if (!path) return {
|
|
2023
|
+
type: "skip",
|
|
2024
|
+
path: "",
|
|
2025
|
+
reason: "not readable: missing path"
|
|
2026
|
+
};
|
|
2027
|
+
let buffer;
|
|
2028
|
+
try {
|
|
2029
|
+
buffer = await readFile(path);
|
|
2030
|
+
} catch (error) {
|
|
2031
|
+
if (isResourceLimitError(error)) throw createResourceLimitError(path, error, options.requestedJobs, options.limits);
|
|
2032
|
+
return {
|
|
2033
|
+
type: "skip",
|
|
2034
|
+
path,
|
|
2035
|
+
reason: `not readable: ${error instanceof Error ? error.message : String(error)}`
|
|
2036
|
+
};
|
|
2037
|
+
}
|
|
2038
|
+
if (isProbablyBinary(buffer)) return {
|
|
2039
|
+
type: "skip",
|
|
2040
|
+
path,
|
|
2041
|
+
reason: "binary file"
|
|
2042
|
+
};
|
|
2043
|
+
return {
|
|
2044
|
+
type: "file",
|
|
2045
|
+
path,
|
|
2046
|
+
content: buffer.toString("utf8")
|
|
2034
2047
|
};
|
|
2035
|
-
await Promise.all(Array.from({ length: concurrency }, () => runWorker()));
|
|
2036
|
-
return results;
|
|
2037
2048
|
}
|
|
2038
2049
|
|
|
2039
2050
|
//#endregion
|
|
2040
|
-
//#region src/cli/batch/jobs/load-count
|
|
2051
|
+
//#region src/cli/batch/jobs/load-count.ts
|
|
2041
2052
|
async function countBatchInputsWithJobs(filePaths, options) {
|
|
2042
2053
|
const limits = resolveBatchJobsLimit();
|
|
2043
2054
|
const total = filePaths.length;
|
|
2044
2055
|
let completed = 0;
|
|
2045
2056
|
const entries = await runBoundedQueue(filePaths.length, options.jobs, async (index) => {
|
|
2046
|
-
const
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
total
|
|
2052
|
-
});
|
|
2053
|
-
return {
|
|
2054
|
-
type: "skip",
|
|
2055
|
-
skip: {
|
|
2056
|
-
path: "",
|
|
2057
|
-
reason: "not readable: missing path"
|
|
2058
|
-
}
|
|
2059
|
-
};
|
|
2060
|
-
}
|
|
2061
|
-
let buffer;
|
|
2062
|
-
try {
|
|
2063
|
-
buffer = await readFile(path);
|
|
2064
|
-
} catch (error) {
|
|
2065
|
-
if (isResourceLimitError(error)) throw createResourceLimitError(path, error, options.jobs, limits);
|
|
2066
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
2067
|
-
completed += 1;
|
|
2068
|
-
options.onFileProcessed?.({
|
|
2069
|
-
completed,
|
|
2070
|
-
total
|
|
2071
|
-
});
|
|
2072
|
-
return {
|
|
2073
|
-
type: "skip",
|
|
2074
|
-
skip: {
|
|
2075
|
-
path,
|
|
2076
|
-
reason: `not readable: ${message}`
|
|
2077
|
-
}
|
|
2078
|
-
};
|
|
2079
|
-
}
|
|
2080
|
-
if (isProbablyBinary(buffer)) {
|
|
2057
|
+
const loaded = await readBatchInput(filePaths[index], {
|
|
2058
|
+
requestedJobs: options.jobs,
|
|
2059
|
+
limits
|
|
2060
|
+
});
|
|
2061
|
+
if (loaded.type === "skip") {
|
|
2081
2062
|
completed += 1;
|
|
2082
2063
|
options.onFileProcessed?.({
|
|
2083
2064
|
completed,
|
|
@@ -2086,13 +2067,12 @@ async function countBatchInputsWithJobs(filePaths, options) {
|
|
|
2086
2067
|
return {
|
|
2087
2068
|
type: "skip",
|
|
2088
2069
|
skip: {
|
|
2089
|
-
path,
|
|
2090
|
-
reason:
|
|
2070
|
+
path: loaded.path,
|
|
2071
|
+
reason: loaded.reason
|
|
2091
2072
|
}
|
|
2092
2073
|
};
|
|
2093
2074
|
}
|
|
2094
|
-
const
|
|
2095
|
-
const result = options.section === "all" ? wc_default(content, options.wcOptions) : countSections(content, options.section, options.wcOptions);
|
|
2075
|
+
const result = options.section === "all" ? wc_default(loaded.content, options.wcOptions) : countSections(loaded.content, options.section, options.wcOptions);
|
|
2096
2076
|
if (!options.preserveCollectorSegments) compactCollectorSegmentsInCountResult(result);
|
|
2097
2077
|
completed += 1;
|
|
2098
2078
|
options.onFileProcessed?.({
|
|
@@ -2102,7 +2082,7 @@ async function countBatchInputsWithJobs(filePaths, options) {
|
|
|
2102
2082
|
return {
|
|
2103
2083
|
type: "file",
|
|
2104
2084
|
file: {
|
|
2105
|
-
path,
|
|
2085
|
+
path: loaded.path,
|
|
2106
2086
|
result
|
|
2107
2087
|
}
|
|
2108
2088
|
};
|
|
@@ -2123,7 +2103,7 @@ async function countBatchInputsWithJobs(filePaths, options) {
|
|
|
2123
2103
|
}
|
|
2124
2104
|
|
|
2125
2105
|
//#endregion
|
|
2126
|
-
//#region src/cli/batch/jobs/load-count-worker
|
|
2106
|
+
//#region src/cli/batch/jobs/load-count-worker.ts
|
|
2127
2107
|
var WorkerRouteUnavailableError = class extends Error {};
|
|
2128
2108
|
function isFallbackFriendlyWorkerError(error) {
|
|
2129
2109
|
if (typeof error !== "object" || error === null) return false;
|
|
@@ -2133,7 +2113,7 @@ function isFallbackFriendlyWorkerError(error) {
|
|
|
2133
2113
|
return message.includes("Unknown file extension") || message.includes("Cannot find module");
|
|
2134
2114
|
}
|
|
2135
2115
|
async function countBatchInputsWithWorkerJobs(filePaths, options) {
|
|
2136
|
-
if (process.env.WORD_COUNTER_DISABLE_EXPERIMENTAL_WORKERS === "1") throw new WorkerRouteUnavailableError("Worker route disabled by environment.");
|
|
2116
|
+
if (process.env.WORD_COUNTER_DISABLE_WORKER_JOBS === "1" || process.env.WORD_COUNTER_DISABLE_EXPERIMENTAL_WORKERS === "1") throw new WorkerRouteUnavailableError("Worker route disabled by environment.");
|
|
2137
2117
|
let workerPoolModule;
|
|
2138
2118
|
try {
|
|
2139
2119
|
workerPoolModule = await import("./worker-pool.mjs");
|
|
@@ -2162,60 +2142,6 @@ async function countBatchInputsWithWorkerJobs(filePaths, options) {
|
|
|
2162
2142
|
}
|
|
2163
2143
|
}
|
|
2164
2144
|
|
|
2165
|
-
//#endregion
|
|
2166
|
-
//#region src/cli/batch/jobs/load-only.ts
|
|
2167
|
-
async function loadBatchInputsWithJobs(filePaths, options) {
|
|
2168
|
-
const limits = resolveBatchJobsLimit();
|
|
2169
|
-
const entries = await runBoundedQueue(filePaths.length, options.jobs, async (index) => {
|
|
2170
|
-
const path = filePaths[index];
|
|
2171
|
-
if (!path) return {
|
|
2172
|
-
type: "skip",
|
|
2173
|
-
path: "",
|
|
2174
|
-
reason: "not readable: missing path"
|
|
2175
|
-
};
|
|
2176
|
-
let buffer;
|
|
2177
|
-
try {
|
|
2178
|
-
buffer = await readFile(path);
|
|
2179
|
-
} catch (error) {
|
|
2180
|
-
if (isResourceLimitError(error)) throw createResourceLimitError(path, error, options.jobs, limits);
|
|
2181
|
-
return {
|
|
2182
|
-
type: "skip",
|
|
2183
|
-
path,
|
|
2184
|
-
reason: `not readable: ${error instanceof Error ? error.message : String(error)}`
|
|
2185
|
-
};
|
|
2186
|
-
}
|
|
2187
|
-
if (isProbablyBinary(buffer)) return {
|
|
2188
|
-
type: "skip",
|
|
2189
|
-
path,
|
|
2190
|
-
reason: "binary file"
|
|
2191
|
-
};
|
|
2192
|
-
return {
|
|
2193
|
-
type: "file",
|
|
2194
|
-
path,
|
|
2195
|
-
content: buffer.toString("utf8")
|
|
2196
|
-
};
|
|
2197
|
-
});
|
|
2198
|
-
const files = [];
|
|
2199
|
-
const skipped = [];
|
|
2200
|
-
for (const entry of entries) {
|
|
2201
|
-
if (entry.type === "file") {
|
|
2202
|
-
files.push({
|
|
2203
|
-
path: entry.path,
|
|
2204
|
-
content: entry.content
|
|
2205
|
-
});
|
|
2206
|
-
continue;
|
|
2207
|
-
}
|
|
2208
|
-
skipped.push({
|
|
2209
|
-
path: entry.path,
|
|
2210
|
-
reason: entry.reason
|
|
2211
|
-
});
|
|
2212
|
-
}
|
|
2213
|
-
return {
|
|
2214
|
-
files,
|
|
2215
|
-
skipped
|
|
2216
|
-
};
|
|
2217
|
-
}
|
|
2218
|
-
|
|
2219
2145
|
//#endregion
|
|
2220
2146
|
//#region src/cli/batch/jobs/render.ts
|
|
2221
2147
|
function finalizeBatchJobsSummary(files, section, wcOptions, options = {}) {
|
|
@@ -2565,165 +2491,115 @@ async function runBatchCount(options) {
|
|
|
2565
2491
|
});
|
|
2566
2492
|
let summary;
|
|
2567
2493
|
let routeSkips = [];
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
options.
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
options.
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2494
|
+
options.debug.emit("batch.load.start", {
|
|
2495
|
+
files: resolved.files.length,
|
|
2496
|
+
jobs: options.jobs,
|
|
2497
|
+
strategy: options.jobsStrategy
|
|
2498
|
+
});
|
|
2499
|
+
options.debug.emit("batch.load.complete", {
|
|
2500
|
+
files: 0,
|
|
2501
|
+
skipped: 0,
|
|
2502
|
+
elapsedMs: 0,
|
|
2503
|
+
strategy: options.jobsStrategy
|
|
2504
|
+
});
|
|
2505
|
+
options.debug.emit("batch.stage.timing", {
|
|
2506
|
+
stage: "load",
|
|
2507
|
+
elapsedMs: 0
|
|
2508
|
+
});
|
|
2509
|
+
const progressEnabled = options.progressReporter.enabled && resolved.files.length > 1;
|
|
2510
|
+
options.debug.emit("batch.progress.start", {
|
|
2511
|
+
enabled: progressEnabled,
|
|
2512
|
+
total: resolved.files.length
|
|
2513
|
+
});
|
|
2514
|
+
if (progressEnabled) options.progressReporter.start(resolved.files.length, batchStartedAtMs);
|
|
2515
|
+
const countStartedAtMs = Date.now();
|
|
2516
|
+
let finalizeStartedAtMs = null;
|
|
2517
|
+
let emittedCountTiming = false;
|
|
2518
|
+
try {
|
|
2519
|
+
let counted;
|
|
2520
|
+
if (options.jobs > 1) try {
|
|
2521
|
+
counted = await countBatchInputsWithWorkerJobs(resolved.files, {
|
|
2522
|
+
jobs: options.jobs,
|
|
2523
|
+
section: options.section,
|
|
2524
|
+
wcOptions: options.wcOptions,
|
|
2525
|
+
preserveCollectorSegments: options.preserveCollectorSegments,
|
|
2526
|
+
onFileProcessed: (snapshot) => {
|
|
2599
2527
|
if (progressEnabled) options.progressReporter.advance(snapshot);
|
|
2600
|
-
}
|
|
2601
|
-
onFinalizeStart: () => {
|
|
2602
|
-
finalizeStartedAtMs = Date.now();
|
|
2603
|
-
if (progressEnabled) options.progressReporter.startFinalizing();
|
|
2604
|
-
const countElapsedMs = finalizeStartedAtMs - countStartedAtMs;
|
|
2605
|
-
options.debug.emit("batch.stage.timing", {
|
|
2606
|
-
stage: "count",
|
|
2607
|
-
elapsedMs: countElapsedMs
|
|
2608
|
-
});
|
|
2609
|
-
emittedCountTiming = true;
|
|
2610
|
-
},
|
|
2611
|
-
preserveCollectorSegments: options.preserveCollectorSegments
|
|
2528
|
+
}
|
|
2612
2529
|
});
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2530
|
+
options.debug.emit("batch.jobs.executor", {
|
|
2531
|
+
strategy: options.jobsStrategy,
|
|
2532
|
+
executor: "worker-pool",
|
|
2533
|
+
jobs: options.jobs
|
|
2534
|
+
});
|
|
2535
|
+
} catch (error) {
|
|
2536
|
+
if (!(error instanceof WorkerRouteUnavailableError)) throw error;
|
|
2537
|
+
options.emitWarning?.(`Worker executor unavailable; falling back to async load+count. (${error.message})`);
|
|
2538
|
+
options.debug.emit("batch.jobs.executor", {
|
|
2539
|
+
strategy: options.jobsStrategy,
|
|
2540
|
+
executor: "async-fallback",
|
|
2541
|
+
reason: error.message,
|
|
2542
|
+
jobs: options.jobs
|
|
2543
|
+
});
|
|
2544
|
+
counted = await countBatchInputsWithJobs(resolved.files, {
|
|
2545
|
+
jobs: options.jobs,
|
|
2546
|
+
section: options.section,
|
|
2547
|
+
wcOptions: options.wcOptions,
|
|
2548
|
+
preserveCollectorSegments: options.preserveCollectorSegments,
|
|
2549
|
+
onFileProcessed: (snapshot) => {
|
|
2550
|
+
if (progressEnabled) options.progressReporter.advance(snapshot);
|
|
2551
|
+
}
|
|
2618
2552
|
});
|
|
2619
2553
|
}
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2554
|
+
else {
|
|
2555
|
+
counted = await countBatchInputsWithJobs(resolved.files, {
|
|
2556
|
+
jobs: options.jobs,
|
|
2557
|
+
section: options.section,
|
|
2558
|
+
wcOptions: options.wcOptions,
|
|
2559
|
+
preserveCollectorSegments: options.preserveCollectorSegments,
|
|
2560
|
+
onFileProcessed: (snapshot) => {
|
|
2561
|
+
if (progressEnabled) options.progressReporter.advance(snapshot);
|
|
2562
|
+
}
|
|
2563
|
+
});
|
|
2564
|
+
options.debug.emit("batch.jobs.executor", {
|
|
2565
|
+
strategy: options.jobsStrategy,
|
|
2566
|
+
executor: "async-main",
|
|
2567
|
+
jobs: options.jobs
|
|
2625
2568
|
});
|
|
2626
2569
|
}
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
files: 0,
|
|
2641
|
-
skipped: 0,
|
|
2642
|
-
elapsedMs: 0,
|
|
2643
|
-
strategy: options.jobsStrategy
|
|
2644
|
-
});
|
|
2645
|
-
options.debug.emit("batch.stage.timing", {
|
|
2646
|
-
stage: "load",
|
|
2647
|
-
elapsedMs: 0
|
|
2570
|
+
routeSkips = counted.skipped;
|
|
2571
|
+
summary = finalizeBatchJobsSummary(counted.files, options.section, options.wcOptions, {
|
|
2572
|
+
onFinalizeStart: () => {
|
|
2573
|
+
finalizeStartedAtMs = Date.now();
|
|
2574
|
+
if (progressEnabled) options.progressReporter.startFinalizing();
|
|
2575
|
+
const countElapsedMs = finalizeStartedAtMs - countStartedAtMs;
|
|
2576
|
+
options.debug.emit("batch.stage.timing", {
|
|
2577
|
+
stage: "count",
|
|
2578
|
+
elapsedMs: countElapsedMs
|
|
2579
|
+
});
|
|
2580
|
+
emittedCountTiming = true;
|
|
2581
|
+
},
|
|
2582
|
+
preserveCollectorSegments: options.preserveCollectorSegments
|
|
2648
2583
|
});
|
|
2649
|
-
|
|
2650
|
-
options.
|
|
2584
|
+
} finally {
|
|
2585
|
+
if (progressEnabled) options.progressReporter.finish();
|
|
2586
|
+
options.debug.emit("batch.progress.complete", {
|
|
2651
2587
|
enabled: progressEnabled,
|
|
2652
2588
|
total: resolved.files.length
|
|
2653
2589
|
});
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
let emittedCountTiming = false;
|
|
2658
|
-
try {
|
|
2659
|
-
let counted;
|
|
2660
|
-
try {
|
|
2661
|
-
counted = await countBatchInputsWithWorkerJobs(resolved.files, {
|
|
2662
|
-
jobs: options.jobs,
|
|
2663
|
-
section: options.section,
|
|
2664
|
-
wcOptions: options.wcOptions,
|
|
2665
|
-
preserveCollectorSegments: options.preserveCollectorSegments,
|
|
2666
|
-
onFileProcessed: (snapshot) => {
|
|
2667
|
-
if (progressEnabled) options.progressReporter.advance(snapshot);
|
|
2668
|
-
}
|
|
2669
|
-
});
|
|
2670
|
-
options.debug.emit("batch.jobs.executor", {
|
|
2671
|
-
strategy: options.jobsStrategy,
|
|
2672
|
-
executor: "worker-pool",
|
|
2673
|
-
jobs: options.jobs
|
|
2674
|
-
});
|
|
2675
|
-
} catch (error) {
|
|
2676
|
-
if (!(error instanceof WorkerRouteUnavailableError)) throw error;
|
|
2677
|
-
options.debug.emit("batch.jobs.executor", {
|
|
2678
|
-
strategy: options.jobsStrategy,
|
|
2679
|
-
executor: "async-fallback",
|
|
2680
|
-
reason: error.message,
|
|
2681
|
-
jobs: options.jobs
|
|
2682
|
-
});
|
|
2683
|
-
counted = await countBatchInputsWithJobs(resolved.files, {
|
|
2684
|
-
jobs: options.jobs,
|
|
2685
|
-
section: options.section,
|
|
2686
|
-
wcOptions: options.wcOptions,
|
|
2687
|
-
preserveCollectorSegments: options.preserveCollectorSegments,
|
|
2688
|
-
onFileProcessed: (snapshot) => {
|
|
2689
|
-
if (progressEnabled) options.progressReporter.advance(snapshot);
|
|
2690
|
-
}
|
|
2691
|
-
});
|
|
2692
|
-
}
|
|
2693
|
-
routeSkips = counted.skipped;
|
|
2694
|
-
summary = finalizeBatchJobsSummary(counted.files, options.section, options.wcOptions, {
|
|
2695
|
-
onFinalizeStart: () => {
|
|
2696
|
-
finalizeStartedAtMs = Date.now();
|
|
2697
|
-
if (progressEnabled) options.progressReporter.startFinalizing();
|
|
2698
|
-
const countElapsedMs = finalizeStartedAtMs - countStartedAtMs;
|
|
2699
|
-
options.debug.emit("batch.stage.timing", {
|
|
2700
|
-
stage: "count",
|
|
2701
|
-
elapsedMs: countElapsedMs
|
|
2702
|
-
});
|
|
2703
|
-
emittedCountTiming = true;
|
|
2704
|
-
},
|
|
2705
|
-
preserveCollectorSegments: options.preserveCollectorSegments
|
|
2706
|
-
});
|
|
2707
|
-
} finally {
|
|
2708
|
-
if (progressEnabled) options.progressReporter.finish();
|
|
2709
|
-
options.debug.emit("batch.progress.complete", {
|
|
2710
|
-
enabled: progressEnabled,
|
|
2711
|
-
total: resolved.files.length
|
|
2712
|
-
});
|
|
2713
|
-
}
|
|
2714
|
-
if (!emittedCountTiming) {
|
|
2715
|
-
const countElapsedMs = Date.now() - countStartedAtMs;
|
|
2716
|
-
options.debug.emit("batch.stage.timing", {
|
|
2717
|
-
stage: "count",
|
|
2718
|
-
elapsedMs: countElapsedMs
|
|
2719
|
-
});
|
|
2720
|
-
}
|
|
2721
|
-
const finalizeElapsedMs = finalizeStartedAtMs === null ? 0 : Date.now() - finalizeStartedAtMs;
|
|
2590
|
+
}
|
|
2591
|
+
if (!emittedCountTiming) {
|
|
2592
|
+
const countElapsedMs = Date.now() - countStartedAtMs;
|
|
2722
2593
|
options.debug.emit("batch.stage.timing", {
|
|
2723
|
-
stage: "
|
|
2724
|
-
elapsedMs:
|
|
2594
|
+
stage: "count",
|
|
2595
|
+
elapsedMs: countElapsedMs
|
|
2725
2596
|
});
|
|
2726
2597
|
}
|
|
2598
|
+
const finalizeElapsedMs = finalizeStartedAtMs === null ? 0 : Date.now() - finalizeStartedAtMs;
|
|
2599
|
+
options.debug.emit("batch.stage.timing", {
|
|
2600
|
+
stage: "finalize",
|
|
2601
|
+
elapsedMs: finalizeElapsedMs
|
|
2602
|
+
});
|
|
2727
2603
|
appendAll(summary.skipped, resolved.skipped);
|
|
2728
2604
|
appendAll(summary.skipped, routeSkips);
|
|
2729
2605
|
options.debug.emit("batch.aggregate.complete", {
|
|
@@ -2736,8 +2612,8 @@ async function runBatchCount(options) {
|
|
|
2736
2612
|
|
|
2737
2613
|
//#endregion
|
|
2738
2614
|
//#region src/cli/batch/jobs/strategy.ts
|
|
2739
|
-
function resolveBatchJobsStrategy(
|
|
2740
|
-
return
|
|
2615
|
+
function resolveBatchJobsStrategy(_jobs) {
|
|
2616
|
+
return "load-count";
|
|
2741
2617
|
}
|
|
2742
2618
|
|
|
2743
2619
|
//#endregion
|
|
@@ -3060,6 +2936,12 @@ function formatInputReadError(error) {
|
|
|
3060
2936
|
//#endregion
|
|
3061
2937
|
//#region src/cli/runtime/batch.ts
|
|
3062
2938
|
async function executeBatchCount({ argv, options, runtime, resolved, debug, teeEnabled }) {
|
|
2939
|
+
const warningsEnabled = !Boolean(options.quietWarnings);
|
|
2940
|
+
const emitWarning = (message) => {
|
|
2941
|
+
if (!warningsEnabled) return;
|
|
2942
|
+
const warningLine = message.startsWith("Warning:") ? message : `Warning: ${message}`;
|
|
2943
|
+
console.error(import_picocolors.default.yellow(warningLine));
|
|
2944
|
+
};
|
|
3063
2945
|
const batchOptions = {
|
|
3064
2946
|
scope: resolveBatchScope(argv),
|
|
3065
2947
|
pathMode: options.pathMode,
|
|
@@ -3071,7 +2953,7 @@ async function executeBatchCount({ argv, options, runtime, resolved, debug, teeE
|
|
|
3071
2953
|
const requestedJobs = options.jobs;
|
|
3072
2954
|
const jobsLimit = resolveBatchJobsLimit();
|
|
3073
2955
|
const jobs = clampRequestedJobs(requestedJobs, jobsLimit);
|
|
3074
|
-
if (requestedJobs > jobsLimit.suggestedMaxJobs)
|
|
2956
|
+
if (requestedJobs > jobsLimit.suggestedMaxJobs) emitWarning(formatJobsAdvisoryWarning(requestedJobs, jobs, jobsLimit));
|
|
3075
2957
|
const jobsStrategy = resolveBatchJobsStrategy(jobs);
|
|
3076
2958
|
const debugEnabled = Boolean(options.debug);
|
|
3077
2959
|
const mirrorDebugToTerminal = debugEnabled && (!debug.reportPath || teeEnabled);
|
|
@@ -3089,16 +2971,19 @@ async function executeBatchCount({ argv, options, runtime, resolved, debug, teeE
|
|
|
3089
2971
|
clearOnFinish: !(mirrorDebugToTerminal || options.keepProgress)
|
|
3090
2972
|
}),
|
|
3091
2973
|
jobs,
|
|
3092
|
-
jobsStrategy
|
|
2974
|
+
jobsStrategy,
|
|
2975
|
+
emitWarning
|
|
3093
2976
|
});
|
|
3094
2977
|
const showSkipDiagnostics = debugEnabled && !batchOptions.quietSkips;
|
|
2978
|
+
const showSkipItems = showSkipDiagnostics && Boolean(options.verbose);
|
|
3095
2979
|
debug.emit("batch.skips.policy", {
|
|
3096
2980
|
enabled: showSkipDiagnostics,
|
|
2981
|
+
items: showSkipItems,
|
|
3097
2982
|
quietSkips: batchOptions.quietSkips
|
|
3098
2983
|
});
|
|
3099
2984
|
if (showSkipDiagnostics) {
|
|
3100
2985
|
debug.emit("batch.skips.report", { count: summary.skipped.length });
|
|
3101
|
-
if (
|
|
2986
|
+
if (showSkipItems) for (const skip of summary.skipped) debug.emit("batch.skips.item", {
|
|
3102
2987
|
path: skip.path,
|
|
3103
2988
|
reason: skip.reason
|
|
3104
2989
|
}, { verbosity: "verbose" });
|