llm-usage-metrics 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -9
- package/dist/index.js +1473 -494
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -106,6 +106,9 @@ function getParsingRuntimeConfig(env = process.env) {
|
|
|
106
106
|
};
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
+
// src/update/update-notifier.ts
|
|
110
|
+
import { spawn as spawn2 } from "child_process";
|
|
111
|
+
|
|
109
112
|
// src/update/update-cache-repository.ts
|
|
110
113
|
import { mkdir, readFile, writeFile } from "fs/promises";
|
|
111
114
|
import path2 from "path";
|
|
@@ -251,8 +254,8 @@ function shouldOfferUpdate(currentVersion, latestVersion) {
|
|
|
251
254
|
}
|
|
252
255
|
|
|
253
256
|
// src/update/update-cache-repository.ts
|
|
254
|
-
var
|
|
255
|
-
var
|
|
257
|
+
var DEFAULT_UPDATE_CHECK_CACHE_TTL_MS = 60 * 60 * 1e3;
|
|
258
|
+
var DEFAULT_UPDATE_CHECK_FETCH_TIMEOUT_MS = 1e3;
|
|
256
259
|
var DEFAULT_FETCH_RETRY_COUNT = 2;
|
|
257
260
|
var DEFAULT_FETCH_RETRY_DELAY_MS = 200;
|
|
258
261
|
var UPDATE_CHECK_CACHE_SCOPE_ENV_VAR = "LLM_USAGE_UPDATE_CACHE_SCOPE";
|
|
@@ -414,8 +417,8 @@ async function fetchLatestVersionWithRetry(packageName2, fetchImpl, fetchTimeout
|
|
|
414
417
|
}
|
|
415
418
|
async function resolveLatestVersion(options) {
|
|
416
419
|
const cacheFilePath = options.cacheFilePath ?? getDefaultUpdateCheckCachePath();
|
|
417
|
-
const cacheTtlMs = options.cacheTtlMs ??
|
|
418
|
-
const fetchTimeoutMs = options.fetchTimeoutMs ??
|
|
420
|
+
const cacheTtlMs = options.cacheTtlMs ?? DEFAULT_UPDATE_CHECK_CACHE_TTL_MS;
|
|
421
|
+
const fetchTimeoutMs = options.fetchTimeoutMs ?? DEFAULT_UPDATE_CHECK_FETCH_TIMEOUT_MS;
|
|
419
422
|
const fetchRetryCount = options.fetchRetryCount ?? DEFAULT_FETCH_RETRY_COUNT;
|
|
420
423
|
const fetchRetryDelayMs = options.fetchRetryDelayMs ?? DEFAULT_FETCH_RETRY_DELAY_MS;
|
|
421
424
|
const fetchImpl = options.fetchImpl ?? fetch;
|
|
@@ -526,6 +529,7 @@ async function runInteractiveInstallAndRestart(options) {
|
|
|
526
529
|
|
|
527
530
|
// src/update/update-notifier.ts
|
|
528
531
|
var UPDATE_CHECK_SKIP_ENV_VAR = "LLM_USAGE_SKIP_UPDATE_CHECK";
|
|
532
|
+
var UPDATE_CHECK_REFRESH_ENV_VAR = "LLM_USAGE_REFRESH_UPDATE_CHECK";
|
|
529
533
|
function isTruthyEnvFlag(value) {
|
|
530
534
|
if (value === void 0) {
|
|
531
535
|
return false;
|
|
@@ -584,6 +588,32 @@ function toResolveLatestVersionOptions(options, env) {
|
|
|
584
588
|
now: options.now
|
|
585
589
|
};
|
|
586
590
|
}
|
|
591
|
+
function runDetachedCommandWithSpawn(command, args, options = {}) {
|
|
592
|
+
const child = spawn2(command, args, {
|
|
593
|
+
env: options.env,
|
|
594
|
+
stdio: options.stdio ?? "ignore",
|
|
595
|
+
detached: true
|
|
596
|
+
});
|
|
597
|
+
child.on("error", () => void 0);
|
|
598
|
+
child.unref();
|
|
599
|
+
}
|
|
600
|
+
function scheduleBackgroundUpdateRefresh(options, env, argv) {
|
|
601
|
+
const spawnDetachedCommand = options.spawnDetachedCommand ?? runDetachedCommandWithSpawn;
|
|
602
|
+
spawnDetachedCommand(options.execPath ?? process.execPath, argv.slice(1), {
|
|
603
|
+
env: {
|
|
604
|
+
...env,
|
|
605
|
+
[UPDATE_CHECK_REFRESH_ENV_VAR]: "1"
|
|
606
|
+
},
|
|
607
|
+
stdio: "ignore"
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
async function refreshUpdateCheckCache(options) {
|
|
611
|
+
try {
|
|
612
|
+
const env = options.env ?? process.env;
|
|
613
|
+
await resolveLatestVersion(toResolveLatestVersionOptions(options, env));
|
|
614
|
+
} catch {
|
|
615
|
+
}
|
|
616
|
+
}
|
|
587
617
|
async function checkForUpdatesAndMaybeRestart(options) {
|
|
588
618
|
const env = options.env ?? process.env;
|
|
589
619
|
const argv = options.argv ?? process.argv;
|
|
@@ -600,7 +630,19 @@ async function checkForUpdatesAndMaybeRestart(options) {
|
|
|
600
630
|
return { continueExecution: true };
|
|
601
631
|
}
|
|
602
632
|
try {
|
|
603
|
-
const
|
|
633
|
+
const resolveOptions = toResolveLatestVersionOptions(options, env);
|
|
634
|
+
const cacheFilePath = resolveOptions.cacheFilePath ?? getDefaultUpdateCheckCachePath();
|
|
635
|
+
const cachePayload = await readUpdateCheckCachePayload(cacheFilePath);
|
|
636
|
+
const cacheTtlMs = resolveOptions.cacheTtlMs ?? DEFAULT_UPDATE_CHECK_CACHE_TTL_MS;
|
|
637
|
+
const now = resolveOptions.now ?? Date.now;
|
|
638
|
+
if (!cachePayload || !isCacheFresh(cachePayload, cacheTtlMs, now)) {
|
|
639
|
+
try {
|
|
640
|
+
scheduleBackgroundUpdateRefresh(options, env, argv);
|
|
641
|
+
} catch {
|
|
642
|
+
}
|
|
643
|
+
return { continueExecution: true };
|
|
644
|
+
}
|
|
645
|
+
const latestVersion = cachePayload.latestVersion;
|
|
604
646
|
if (!latestVersion || !shouldOfferUpdate(options.currentVersion, latestVersion)) {
|
|
605
647
|
return { continueExecution: true };
|
|
606
648
|
}
|
|
@@ -2260,8 +2302,9 @@ function addOutcomeTotals(left, right) {
|
|
|
2260
2302
|
};
|
|
2261
2303
|
}
|
|
2262
2304
|
function addUsageTotals(left, right) {
|
|
2305
|
+
const hasAnyBucketUsage = (value) => value.inputTokens > 0 || value.outputTokens > 0 || value.reasoningTokens > 0 || value.cacheReadTokens > 0 || value.cacheWriteTokens > 0;
|
|
2263
2306
|
const hasUnknownCost = left.costIncomplete === true && left.costUsd === void 0 || right.costIncomplete === true && right.costUsd === void 0;
|
|
2264
|
-
const isNeutralZeroCost = (value) => value.totalTokens === 0 && value.costUsd === 0 && value.costIncomplete !== true;
|
|
2307
|
+
const isNeutralZeroCost = (value) => !hasAnyBucketUsage(value) && value.totalTokens === 0 && value.costUsd === 0 && value.costIncomplete !== true;
|
|
2265
2308
|
const leftKnownCost = left.costUsd !== void 0 && !isNeutralZeroCost(left) ? left.costUsd : void 0;
|
|
2266
2309
|
const rightKnownCost = right.costUsd !== void 0 && !isNeutralZeroCost(right) ? right.costUsd : void 0;
|
|
2267
2310
|
let costUsd = leftKnownCost !== void 0 && rightKnownCost !== void 0 ? addUsd2(leftKnownCost, rightKnownCost) : leftKnownCost ?? rightKnownCost;
|
|
@@ -2279,6 +2322,9 @@ function addUsageTotals(left, right) {
|
|
|
2279
2322
|
costIncomplete: left.costIncomplete || right.costIncomplete ? true : void 0
|
|
2280
2323
|
};
|
|
2281
2324
|
}
|
|
2325
|
+
function hasMeaningfulUsageSignal(usageTotals) {
|
|
2326
|
+
return usageTotals.totalTokens > 0 || usageTotals.inputTokens > 0 || usageTotals.outputTokens > 0 || usageTotals.reasoningTokens > 0 || usageTotals.cacheReadTokens > 0 || usageTotals.cacheWriteTokens > 0 || usageTotals.costUsd !== void 0 || usageTotals.costIncomplete === true;
|
|
2327
|
+
}
|
|
2282
2328
|
function computeDerivedMetrics(usage, outcomes) {
|
|
2283
2329
|
const costUsd = usage.costUsd;
|
|
2284
2330
|
const nonCacheTotalTokens = usage.inputTokens + usage.outputTokens + usage.reasoningTokens;
|
|
@@ -2301,7 +2347,7 @@ function aggregateEfficiency(options) {
|
|
|
2301
2347
|
const usageTotals = usageTotalsByPeriod.get(periodKey) ?? createEmptyEfficiencyUsageTotals();
|
|
2302
2348
|
const outcomeTotals = options.periodOutcomes.get(periodKey) ?? createEmptyEfficiencyOutcomeTotals();
|
|
2303
2349
|
const hasUsageRow = usageTotalsByPeriod.has(periodKey);
|
|
2304
|
-
const hasUsageSignal3 = hasUsageRow && (usageTotals
|
|
2350
|
+
const hasUsageSignal3 = hasUsageRow && hasMeaningfulUsageSignal(usageTotals);
|
|
2305
2351
|
if (outcomeTotals.commitCount === 0 || !hasUsageSignal3) {
|
|
2306
2352
|
continue;
|
|
2307
2353
|
}
|
|
@@ -2328,7 +2374,7 @@ function aggregateEfficiency(options) {
|
|
|
2328
2374
|
}
|
|
2329
2375
|
|
|
2330
2376
|
// src/efficiency/git-outcome-collector.ts
|
|
2331
|
-
import { spawn as
|
|
2377
|
+
import { spawn as spawn3 } from "child_process";
|
|
2332
2378
|
import { createInterface as createInterface2 } from "readline";
|
|
2333
2379
|
import path3 from "path";
|
|
2334
2380
|
import { stat } from "fs/promises";
|
|
@@ -2527,7 +2573,7 @@ function parseGitLogShortstatLines(lines, authorEmail) {
|
|
|
2527
2573
|
}
|
|
2528
2574
|
async function runGitCommand(repoDir, args) {
|
|
2529
2575
|
return await new Promise((resolve, reject) => {
|
|
2530
|
-
const child =
|
|
2576
|
+
const child = spawn3("git", args, {
|
|
2531
2577
|
cwd: repoDir,
|
|
2532
2578
|
env: {
|
|
2533
2579
|
...process.env,
|
|
@@ -2812,86 +2858,6 @@ async function attributeUsageEventsToRepo(events, repoDir, resolveRepoRoot3 = re
|
|
|
2812
2858
|
};
|
|
2813
2859
|
}
|
|
2814
2860
|
|
|
2815
|
-
// src/cli/build-usage-data-diagnostics.ts
|
|
2816
|
-
function buildUsageDiagnostics(params) {
|
|
2817
|
-
const parseResultBySource = new Map(
|
|
2818
|
-
params.successfulParseResults.map((result) => [result.source.toLowerCase(), result])
|
|
2819
|
-
);
|
|
2820
|
-
const sessionStats = params.adaptersToParse.map((adapter) => {
|
|
2821
|
-
const parseResult = parseResultBySource.get(adapter.id.toLowerCase());
|
|
2822
|
-
return {
|
|
2823
|
-
source: adapter.id,
|
|
2824
|
-
filesFound: parseResult?.filesFound ?? 0,
|
|
2825
|
-
eventsParsed: parseResult?.events.length ?? 0
|
|
2826
|
-
};
|
|
2827
|
-
});
|
|
2828
|
-
const skippedRows = params.successfulParseResults.filter((result) => result.skippedRows > 0).map((result) => ({
|
|
2829
|
-
source: result.source,
|
|
2830
|
-
skippedRows: result.skippedRows,
|
|
2831
|
-
reasons: result.skippedRowReasons
|
|
2832
|
-
}));
|
|
2833
|
-
return {
|
|
2834
|
-
sessionStats,
|
|
2835
|
-
sourceFailures: params.sourceFailures,
|
|
2836
|
-
skippedRows,
|
|
2837
|
-
pricingOrigin: params.pricingOrigin,
|
|
2838
|
-
pricingWarning: params.pricingWarning,
|
|
2839
|
-
activeEnvOverrides: params.activeEnvOverrides,
|
|
2840
|
-
timezone: params.timezone
|
|
2841
|
-
};
|
|
2842
|
-
}
|
|
2843
|
-
function assembleUsageDataResult(events, rows, diagnostics) {
|
|
2844
|
-
return {
|
|
2845
|
-
events,
|
|
2846
|
-
rows,
|
|
2847
|
-
diagnostics
|
|
2848
|
-
};
|
|
2849
|
-
}
|
|
2850
|
-
|
|
2851
|
-
// src/config/env-var-display.ts
|
|
2852
|
-
var ENV_VARS_TO_DISPLAY = [
|
|
2853
|
-
{ name: "LLM_USAGE_SKIP_UPDATE_CHECK", description: "skip startup update check" },
|
|
2854
|
-
{
|
|
2855
|
-
name: "LLM_USAGE_UPDATE_CACHE_SCOPE",
|
|
2856
|
-
description: "update-check cache scope (global/session)"
|
|
2857
|
-
},
|
|
2858
|
-
{ name: "LLM_USAGE_UPDATE_CACHE_SESSION_KEY", description: "update-check session cache key" },
|
|
2859
|
-
{ name: "LLM_USAGE_UPDATE_CACHE_TTL_MS", description: "update-check cache TTL" },
|
|
2860
|
-
{ name: "LLM_USAGE_UPDATE_FETCH_TIMEOUT_MS", description: "update-check fetch timeout" },
|
|
2861
|
-
{ name: "LLM_USAGE_PRICING_CACHE_TTL_MS", description: "pricing cache TTL" },
|
|
2862
|
-
{ name: "LLM_USAGE_PRICING_FETCH_TIMEOUT_MS", description: "pricing fetch timeout" },
|
|
2863
|
-
{ name: "LLM_USAGE_PARSE_MAX_PARALLEL", description: "max parallel file parsing" },
|
|
2864
|
-
{ name: "LLM_USAGE_PARSE_CACHE_ENABLED", description: "enable file parse cache" },
|
|
2865
|
-
{ name: "LLM_USAGE_PARSE_CACHE_TTL_MS", description: "file parse cache TTL" },
|
|
2866
|
-
{ name: "LLM_USAGE_PARSE_CACHE_MAX_ENTRIES", description: "file parse cache max entries" },
|
|
2867
|
-
{ name: "LLM_USAGE_PARSE_CACHE_MAX_BYTES", description: "file parse cache max bytes" }
|
|
2868
|
-
];
|
|
2869
|
-
function getActiveEnvVarOverrides() {
|
|
2870
|
-
const overrides = [];
|
|
2871
|
-
for (const { name, description } of ENV_VARS_TO_DISPLAY) {
|
|
2872
|
-
const value = process.env[name];
|
|
2873
|
-
if (value !== void 0 && value !== "") {
|
|
2874
|
-
overrides.push({ name, value, description });
|
|
2875
|
-
}
|
|
2876
|
-
}
|
|
2877
|
-
return overrides;
|
|
2878
|
-
}
|
|
2879
|
-
function formatEnvVarOverrides(overrides) {
|
|
2880
|
-
if (overrides.length === 0) {
|
|
2881
|
-
return [];
|
|
2882
|
-
}
|
|
2883
|
-
const lines = [];
|
|
2884
|
-
lines.push("Active environment overrides:");
|
|
2885
|
-
for (const { name, value, description } of overrides) {
|
|
2886
|
-
lines.push(` ${name}=${value} (${description})`);
|
|
2887
|
-
}
|
|
2888
|
-
return lines;
|
|
2889
|
-
}
|
|
2890
|
-
|
|
2891
|
-
// src/sources/codex/codex-source-adapter.ts
|
|
2892
|
-
import os2 from "os";
|
|
2893
|
-
import path6 from "path";
|
|
2894
|
-
|
|
2895
2861
|
// src/domain/provider-normalization.ts
|
|
2896
2862
|
var billingProviderAliases = /* @__PURE__ */ new Map([
|
|
2897
2863
|
["openai-codex", "openai"],
|
|
@@ -2901,6 +2867,13 @@ var billingProviderPrefixAliases = [
|
|
|
2901
2867
|
["openai-", "openai"],
|
|
2902
2868
|
["openai/", "openai"]
|
|
2903
2869
|
];
|
|
2870
|
+
var knownCanonicalProviderRoots = /* @__PURE__ */ new Set(["anthropic", "github", "google", "openai"]);
|
|
2871
|
+
var explicitModelProviderRootPatterns = [
|
|
2872
|
+
[/^gpt-/u, "openai"],
|
|
2873
|
+
[/^o(?:1|3|4)(?:$|[-.])/u, "openai"],
|
|
2874
|
+
[/^claude(?:$|[-.])/u, "anthropic"],
|
|
2875
|
+
[/^gemini(?:$|[-.])/u, "google"]
|
|
2876
|
+
];
|
|
2904
2877
|
function normalizeProviderToBillingEntity(provider) {
|
|
2905
2878
|
if (!provider) {
|
|
2906
2879
|
return void 0;
|
|
@@ -2920,6 +2893,73 @@ function normalizeProviderToBillingEntity(provider) {
|
|
|
2920
2893
|
}
|
|
2921
2894
|
return normalizedProvider;
|
|
2922
2895
|
}
|
|
2896
|
+
function matchesCanonicalProviderFilter(provider, providerFilter) {
|
|
2897
|
+
const normalizedFilter = normalizeProviderToBillingEntity(providerFilter);
|
|
2898
|
+
if (!normalizedFilter) {
|
|
2899
|
+
return true;
|
|
2900
|
+
}
|
|
2901
|
+
const normalizedProvider = normalizeProviderToBillingEntity(provider);
|
|
2902
|
+
if (!normalizedProvider) {
|
|
2903
|
+
return false;
|
|
2904
|
+
}
|
|
2905
|
+
return normalizedProvider.includes(normalizedFilter);
|
|
2906
|
+
}
|
|
2907
|
+
function collectCanonicalProviderRoots(providers) {
|
|
2908
|
+
const canonicalProviders = /* @__PURE__ */ new Set();
|
|
2909
|
+
for (const provider of providers) {
|
|
2910
|
+
const normalizedProvider = normalizeProviderToBillingEntity(provider);
|
|
2911
|
+
if (normalizedProvider) {
|
|
2912
|
+
canonicalProviders.add(normalizedProvider);
|
|
2913
|
+
}
|
|
2914
|
+
}
|
|
2915
|
+
return [...canonicalProviders].sort(compareByCodePoint);
|
|
2916
|
+
}
|
|
2917
|
+
function resolveExplicitProviderRoots(providerFilter) {
|
|
2918
|
+
if (!providerFilter) {
|
|
2919
|
+
return void 0;
|
|
2920
|
+
}
|
|
2921
|
+
const normalizedProviderFilter = normalizeProviderToBillingEntity(providerFilter);
|
|
2922
|
+
if (!normalizedProviderFilter || !knownCanonicalProviderRoots.has(normalizedProviderFilter)) {
|
|
2923
|
+
return void 0;
|
|
2924
|
+
}
|
|
2925
|
+
return [normalizedProviderFilter];
|
|
2926
|
+
}
|
|
2927
|
+
function inferCanonicalProviderRootFromModel(model) {
|
|
2928
|
+
const normalizedModel = model.trim().toLowerCase();
|
|
2929
|
+
if (!normalizedModel) {
|
|
2930
|
+
return void 0;
|
|
2931
|
+
}
|
|
2932
|
+
for (const [pattern, providerRoot] of explicitModelProviderRootPatterns) {
|
|
2933
|
+
if (pattern.test(normalizedModel)) {
|
|
2934
|
+
return providerRoot;
|
|
2935
|
+
}
|
|
2936
|
+
}
|
|
2937
|
+
return void 0;
|
|
2938
|
+
}
|
|
2939
|
+
function inferCanonicalProviderRootsFromModels(models) {
|
|
2940
|
+
if (!models || models.length === 0) {
|
|
2941
|
+
return void 0;
|
|
2942
|
+
}
|
|
2943
|
+
const inferredProviderRoots = /* @__PURE__ */ new Set();
|
|
2944
|
+
for (const model of models) {
|
|
2945
|
+
const inferredProviderRoot = inferCanonicalProviderRootFromModel(model);
|
|
2946
|
+
if (!inferredProviderRoot) {
|
|
2947
|
+
return void 0;
|
|
2948
|
+
}
|
|
2949
|
+
inferredProviderRoots.add(inferredProviderRoot);
|
|
2950
|
+
}
|
|
2951
|
+
return [...inferredProviderRoots].sort(compareByCodePoint);
|
|
2952
|
+
}
|
|
2953
|
+
function intersectCanonicalProviderRoots(left, right) {
|
|
2954
|
+
if (!left) {
|
|
2955
|
+
return right;
|
|
2956
|
+
}
|
|
2957
|
+
if (!right) {
|
|
2958
|
+
return left;
|
|
2959
|
+
}
|
|
2960
|
+
const rightSet = new Set(right);
|
|
2961
|
+
return left.filter((providerRoot) => rightSet.has(providerRoot));
|
|
2962
|
+
}
|
|
2923
2963
|
|
|
2924
2964
|
// src/domain/usage-event.ts
|
|
2925
2965
|
function normalizeSourceId(value) {
|
|
@@ -2977,7 +3017,7 @@ function createUsageEvent(input) {
|
|
|
2977
3017
|
const cacheWriteTokens = normalizeNonNegativeInteger(input.cacheWriteTokens);
|
|
2978
3018
|
const declaredTotalTokens = normalizeNonNegativeInteger(input.totalTokens);
|
|
2979
3019
|
const componentTotalTokens = inputTokens + outputTokens + reasoningTokens + cacheReadTokens + cacheWriteTokens;
|
|
2980
|
-
const totalTokens =
|
|
3020
|
+
const totalTokens = input.totalTokens === void 0 ? componentTotalTokens : declaredTotalTokens;
|
|
2981
3021
|
const costUsd = normalizeUsdCost(input.costUsd);
|
|
2982
3022
|
const costMode = resolveCostMode(input.costMode, costUsd);
|
|
2983
3023
|
return {
|
|
@@ -2997,9 +3037,97 @@ function createUsageEvent(input) {
|
|
|
2997
3037
|
costMode
|
|
2998
3038
|
};
|
|
2999
3039
|
}
|
|
3040
|
+
function hasBillableTokenBuckets(usage) {
|
|
3041
|
+
return usage.inputTokens > 0 || usage.outputTokens > 0 || usage.reasoningTokens > 0 || usage.cacheReadTokens > 0 || usage.cacheWriteTokens > 0;
|
|
3042
|
+
}
|
|
3043
|
+
function isPriceableEvent(event) {
|
|
3044
|
+
return hasBillableTokenBuckets(event);
|
|
3045
|
+
}
|
|
3046
|
+
|
|
3047
|
+
// src/cli/build-usage-data-diagnostics.ts
|
|
3048
|
+
function buildUsageDiagnostics(params) {
|
|
3049
|
+
const parseResultBySource = new Map(
|
|
3050
|
+
params.successfulParseResults.map((result) => [result.source.toLowerCase(), result])
|
|
3051
|
+
);
|
|
3052
|
+
const sessionStats = params.adaptersToParse.map((adapter) => {
|
|
3053
|
+
const parseResult = parseResultBySource.get(adapter.id.toLowerCase());
|
|
3054
|
+
return {
|
|
3055
|
+
source: adapter.id,
|
|
3056
|
+
filesFound: parseResult?.filesFound ?? 0,
|
|
3057
|
+
eventsParsed: parseResult?.events.length ?? 0
|
|
3058
|
+
};
|
|
3059
|
+
});
|
|
3060
|
+
const skippedRows = params.successfulParseResults.filter((result) => result.skippedRows > 0).map((result) => ({
|
|
3061
|
+
source: result.source,
|
|
3062
|
+
skippedRows: result.skippedRows,
|
|
3063
|
+
reasons: result.skippedRowReasons
|
|
3064
|
+
}));
|
|
3065
|
+
return {
|
|
3066
|
+
sessionStats,
|
|
3067
|
+
sourceFailures: params.sourceFailures,
|
|
3068
|
+
skippedRows,
|
|
3069
|
+
pricingOrigin: params.pricingOrigin,
|
|
3070
|
+
pricingWarning: params.pricingWarning,
|
|
3071
|
+
activeEnvOverrides: params.activeEnvOverrides,
|
|
3072
|
+
timezone: params.timezone,
|
|
3073
|
+
runtimeProfile: params.runtimeProfile
|
|
3074
|
+
};
|
|
3075
|
+
}
|
|
3076
|
+
function assembleUsageDataResult(events, rows, diagnostics) {
|
|
3077
|
+
return {
|
|
3078
|
+
events,
|
|
3079
|
+
rows,
|
|
3080
|
+
diagnostics
|
|
3081
|
+
};
|
|
3082
|
+
}
|
|
3083
|
+
|
|
3084
|
+
// src/config/env-var-display.ts
|
|
3085
|
+
var ENV_VARS_TO_DISPLAY = [
|
|
3086
|
+
{ name: "LLM_USAGE_SKIP_UPDATE_CHECK", description: "skip startup update check" },
|
|
3087
|
+
{
|
|
3088
|
+
name: "LLM_USAGE_UPDATE_CACHE_SCOPE",
|
|
3089
|
+
description: "update-check cache scope (global/session)"
|
|
3090
|
+
},
|
|
3091
|
+
{ name: "LLM_USAGE_UPDATE_CACHE_SESSION_KEY", description: "update-check session cache key" },
|
|
3092
|
+
{ name: "LLM_USAGE_UPDATE_CACHE_TTL_MS", description: "update-check cache TTL" },
|
|
3093
|
+
{ name: "LLM_USAGE_UPDATE_FETCH_TIMEOUT_MS", description: "update-check fetch timeout" },
|
|
3094
|
+
{ name: "LLM_USAGE_PRICING_CACHE_TTL_MS", description: "pricing cache TTL" },
|
|
3095
|
+
{ name: "LLM_USAGE_PRICING_FETCH_TIMEOUT_MS", description: "pricing fetch timeout" },
|
|
3096
|
+
{ name: "LLM_USAGE_PARSE_MAX_PARALLEL", description: "max parallel file parsing" },
|
|
3097
|
+
{ name: "LLM_USAGE_PARSE_CACHE_ENABLED", description: "enable file parse cache" },
|
|
3098
|
+
{ name: "LLM_USAGE_PARSE_CACHE_TTL_MS", description: "file parse cache TTL" },
|
|
3099
|
+
{ name: "LLM_USAGE_PARSE_CACHE_MAX_ENTRIES", description: "file parse cache max entries" },
|
|
3100
|
+
{ name: "LLM_USAGE_PARSE_CACHE_MAX_BYTES", description: "file parse cache max bytes" },
|
|
3101
|
+
{ name: "LLM_USAGE_PROFILE_RUNTIME", description: "emit runtime profiling diagnostics" }
|
|
3102
|
+
];
|
|
3103
|
+
function getActiveEnvVarOverrides() {
|
|
3104
|
+
const overrides = [];
|
|
3105
|
+
for (const { name, description } of ENV_VARS_TO_DISPLAY) {
|
|
3106
|
+
const value = process.env[name];
|
|
3107
|
+
if (value !== void 0 && value !== "") {
|
|
3108
|
+
overrides.push({ name, value, description });
|
|
3109
|
+
}
|
|
3110
|
+
}
|
|
3111
|
+
return overrides;
|
|
3112
|
+
}
|
|
3113
|
+
function formatEnvVarOverrides(overrides) {
|
|
3114
|
+
if (overrides.length === 0) {
|
|
3115
|
+
return [];
|
|
3116
|
+
}
|
|
3117
|
+
const lines = [];
|
|
3118
|
+
lines.push("Active environment overrides:");
|
|
3119
|
+
for (const { name, value, description } of overrides) {
|
|
3120
|
+
lines.push(` ${name}=${value} (${description})`);
|
|
3121
|
+
}
|
|
3122
|
+
return lines;
|
|
3123
|
+
}
|
|
3124
|
+
|
|
3125
|
+
// src/sources/codex/codex-source-adapter.ts
|
|
3126
|
+
import os2 from "os";
|
|
3127
|
+
import path6 from "path";
|
|
3000
3128
|
|
|
3001
3129
|
// src/utils/discover-files.ts
|
|
3002
|
-
import { readdir } from "fs/promises";
|
|
3130
|
+
import { readdir, realpath as realpath2, stat as stat2 } from "fs/promises";
|
|
3003
3131
|
import path5 from "path";
|
|
3004
3132
|
function getNodeErrorCode2(error) {
|
|
3005
3133
|
const record = asRecord(error);
|
|
@@ -3024,7 +3152,22 @@ function normalizeExtension(extension) {
|
|
|
3024
3152
|
}
|
|
3025
3153
|
return normalized;
|
|
3026
3154
|
}
|
|
3027
|
-
async function walkDirectory(rootDir, acc, options) {
|
|
3155
|
+
async function walkDirectory(rootDir, acc, options, ancestryRealPaths) {
|
|
3156
|
+
let resolvedRootDir;
|
|
3157
|
+
try {
|
|
3158
|
+
resolvedRootDir = await realpath2(rootDir);
|
|
3159
|
+
} catch (error) {
|
|
3160
|
+
if (getNodeErrorCode2(error) === "ENOENT") {
|
|
3161
|
+
return;
|
|
3162
|
+
}
|
|
3163
|
+
if (options.allowPermissionSkip && isSkippableDirectoryReadError(error)) {
|
|
3164
|
+
return;
|
|
3165
|
+
}
|
|
3166
|
+
throw error;
|
|
3167
|
+
}
|
|
3168
|
+
if (ancestryRealPaths.has(resolvedRootDir)) {
|
|
3169
|
+
return;
|
|
3170
|
+
}
|
|
3028
3171
|
let entries;
|
|
3029
3172
|
try {
|
|
3030
3173
|
entries = await readdir(rootDir, { withFileTypes: true, encoding: "utf8" });
|
|
@@ -3037,13 +3180,37 @@ async function walkDirectory(rootDir, acc, options) {
|
|
|
3037
3180
|
}
|
|
3038
3181
|
throw error;
|
|
3039
3182
|
}
|
|
3183
|
+
const nextAncestryRealPaths = new Set(ancestryRealPaths);
|
|
3184
|
+
nextAncestryRealPaths.add(resolvedRootDir);
|
|
3040
3185
|
if (options.sort) {
|
|
3041
3186
|
entries.sort((left, right) => compareByCodePoint(left.name, right.name));
|
|
3042
3187
|
}
|
|
3043
3188
|
for (const entry of entries) {
|
|
3044
3189
|
const entryPath = path5.join(rootDir, entry.name);
|
|
3190
|
+
if (entry.isSymbolicLink()) {
|
|
3191
|
+
try {
|
|
3192
|
+
const entryStats = await stat2(entryPath);
|
|
3193
|
+
const resolvedEntryPath = await realpath2(entryPath);
|
|
3194
|
+
if (entryStats.isDirectory() && options.recursive) {
|
|
3195
|
+
await walkDirectory(entryPath, acc, options, nextAncestryRealPaths);
|
|
3196
|
+
continue;
|
|
3197
|
+
}
|
|
3198
|
+
if (entryStats.isFile() && (matchesExtension(entry.name, options.extension) || matchesExtension(path5.basename(resolvedEntryPath), options.extension))) {
|
|
3199
|
+
acc.push(entryPath);
|
|
3200
|
+
}
|
|
3201
|
+
} catch (error) {
|
|
3202
|
+
if (getNodeErrorCode2(error) === "ENOENT") {
|
|
3203
|
+
continue;
|
|
3204
|
+
}
|
|
3205
|
+
if (options.allowPermissionSkip && isSkippableDirectoryReadError(error)) {
|
|
3206
|
+
continue;
|
|
3207
|
+
}
|
|
3208
|
+
throw error;
|
|
3209
|
+
}
|
|
3210
|
+
continue;
|
|
3211
|
+
}
|
|
3045
3212
|
if (entry.isDirectory() && options.recursive) {
|
|
3046
|
-
await walkDirectory(entryPath, acc, options);
|
|
3213
|
+
await walkDirectory(entryPath, acc, options, nextAncestryRealPaths);
|
|
3047
3214
|
continue;
|
|
3048
3215
|
}
|
|
3049
3216
|
if (entry.isFile() && matchesExtension(entry.name, options.extension)) {
|
|
@@ -3051,6 +3218,33 @@ async function walkDirectory(rootDir, acc, options) {
|
|
|
3051
3218
|
}
|
|
3052
3219
|
}
|
|
3053
3220
|
}
|
|
3221
|
+
async function toCanonicalFiles(files, options) {
|
|
3222
|
+
const canonicalFiles = [];
|
|
3223
|
+
const seenRealPaths = /* @__PURE__ */ new Set();
|
|
3224
|
+
for (const filePath of files) {
|
|
3225
|
+
let resolvedFilePath;
|
|
3226
|
+
try {
|
|
3227
|
+
resolvedFilePath = await realpath2(filePath);
|
|
3228
|
+
} catch (error) {
|
|
3229
|
+
if (getNodeErrorCode2(error) === "ENOENT") {
|
|
3230
|
+
continue;
|
|
3231
|
+
}
|
|
3232
|
+
if (options.allowPermissionSkip && isSkippableDirectoryReadError(error)) {
|
|
3233
|
+
continue;
|
|
3234
|
+
}
|
|
3235
|
+
throw error;
|
|
3236
|
+
}
|
|
3237
|
+
if (seenRealPaths.has(resolvedFilePath)) {
|
|
3238
|
+
continue;
|
|
3239
|
+
}
|
|
3240
|
+
seenRealPaths.add(resolvedFilePath);
|
|
3241
|
+
canonicalFiles.push(resolvedFilePath);
|
|
3242
|
+
}
|
|
3243
|
+
if (options.sort) {
|
|
3244
|
+
canonicalFiles.sort(compareByCodePoint);
|
|
3245
|
+
}
|
|
3246
|
+
return canonicalFiles;
|
|
3247
|
+
}
|
|
3054
3248
|
async function discoverFiles(rootDir, options) {
|
|
3055
3249
|
const files = [];
|
|
3056
3250
|
const resolvedOptions = {
|
|
@@ -3059,8 +3253,8 @@ async function discoverFiles(rootDir, options) {
|
|
|
3059
3253
|
allowPermissionSkip: options.allowPermissionSkip ?? true,
|
|
3060
3254
|
sort: options.sort ?? true
|
|
3061
3255
|
};
|
|
3062
|
-
await walkDirectory(rootDir, files, resolvedOptions);
|
|
3063
|
-
return files;
|
|
3256
|
+
await walkDirectory(rootDir, files, resolvedOptions, /* @__PURE__ */ new Set());
|
|
3257
|
+
return toCanonicalFiles(files, resolvedOptions);
|
|
3064
3258
|
}
|
|
3065
3259
|
|
|
3066
3260
|
// src/utils/discover-jsonl-files.ts
|
|
@@ -3069,7 +3263,7 @@ async function discoverJsonlFiles(rootDir) {
|
|
|
3069
3263
|
}
|
|
3070
3264
|
|
|
3071
3265
|
// src/utils/fs-helpers.ts
|
|
3072
|
-
import { access as access2, constants as constants2, stat as
|
|
3266
|
+
import { access as access2, constants as constants2, stat as stat3 } from "fs/promises";
|
|
3073
3267
|
async function pathExists(filePath) {
|
|
3074
3268
|
try {
|
|
3075
3269
|
await access2(filePath, constants2.F_OK);
|
|
@@ -3088,21 +3282,21 @@ async function pathReadable(filePath) {
|
|
|
3088
3282
|
}
|
|
3089
3283
|
async function pathIsDirectory(filePath) {
|
|
3090
3284
|
try {
|
|
3091
|
-
return (await
|
|
3285
|
+
return (await stat3(filePath)).isDirectory();
|
|
3092
3286
|
} catch {
|
|
3093
3287
|
return false;
|
|
3094
3288
|
}
|
|
3095
3289
|
}
|
|
3096
3290
|
async function pathIsFile(filePath) {
|
|
3097
3291
|
try {
|
|
3098
|
-
return (await
|
|
3292
|
+
return (await stat3(filePath)).isFile();
|
|
3099
3293
|
} catch {
|
|
3100
3294
|
return false;
|
|
3101
3295
|
}
|
|
3102
3296
|
}
|
|
3103
3297
|
async function pathStat(filePath) {
|
|
3104
3298
|
try {
|
|
3105
|
-
return await
|
|
3299
|
+
return await stat3(filePath);
|
|
3106
3300
|
} catch {
|
|
3107
3301
|
return void 0;
|
|
3108
3302
|
}
|
|
@@ -3150,6 +3344,8 @@ async function* readJsonlObjects(filePath, options = {}) {
|
|
|
3150
3344
|
}
|
|
3151
3345
|
|
|
3152
3346
|
// src/sources/parsing-utils.ts
|
|
3347
|
+
var MIN_PLAUSIBLE_UNIX_SECONDS_ABS = 1e8;
|
|
3348
|
+
var UNIX_SECONDS_ABS_CUTOFF = 1e10;
|
|
3153
3349
|
function asTrimmedText(value) {
|
|
3154
3350
|
if (typeof value !== "string") {
|
|
3155
3351
|
return void 0;
|
|
@@ -3166,13 +3362,45 @@ function toNumberLike(value) {
|
|
|
3166
3362
|
}
|
|
3167
3363
|
return void 0;
|
|
3168
3364
|
}
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3365
|
+
function normalizeTimestampCandidate(candidate) {
|
|
3366
|
+
if (candidate instanceof Date) {
|
|
3367
|
+
return Number.isNaN(candidate.getTime()) ? void 0 : candidate.toISOString();
|
|
3368
|
+
}
|
|
3369
|
+
if (typeof candidate === "number" && Number.isFinite(candidate)) {
|
|
3370
|
+
if (Math.abs(candidate) < MIN_PLAUSIBLE_UNIX_SECONDS_ABS) {
|
|
3371
|
+
return void 0;
|
|
3372
|
+
}
|
|
3373
|
+
const timestampMs = Math.abs(candidate) <= UNIX_SECONDS_ABS_CUTOFF ? candidate * 1e3 : candidate;
|
|
3374
|
+
const date2 = new Date(timestampMs);
|
|
3375
|
+
if (Number.isNaN(date2.getTime())) {
|
|
3376
|
+
return void 0;
|
|
3377
|
+
}
|
|
3378
|
+
return date2.toISOString();
|
|
3379
|
+
}
|
|
3380
|
+
const normalizedText = asTrimmedText(candidate);
|
|
3381
|
+
if (!normalizedText) {
|
|
3382
|
+
return void 0;
|
|
3383
|
+
}
|
|
3384
|
+
const numericTimestamp = /^-?\d+$/u.test(normalizedText) && normalizedText.length >= 9 ? Number(normalizedText) : NaN;
|
|
3385
|
+
if (Number.isFinite(numericTimestamp)) {
|
|
3386
|
+
return normalizeTimestampCandidate(numericTimestamp);
|
|
3387
|
+
}
|
|
3388
|
+
if (/^-?\d+$/u.test(normalizedText)) {
|
|
3389
|
+
return void 0;
|
|
3390
|
+
}
|
|
3391
|
+
const date = new Date(normalizedText);
|
|
3392
|
+
if (Number.isNaN(date.getTime())) {
|
|
3393
|
+
return void 0;
|
|
3394
|
+
}
|
|
3395
|
+
return date.toISOString();
|
|
3396
|
+
}
|
|
3397
|
+
|
|
3398
|
+
// src/sources/codex/codex-source-adapter.ts
|
|
3399
|
+
var defaultSessionsDir = path6.join(os2.homedir(), ".codex", "sessions");
|
|
3400
|
+
var LEGACY_CODEX_MODEL_FALLBACK = "legacy-codex-unknown";
|
|
3401
|
+
var SESSION_META_LINE_PATTERN = /"type"\s*:\s*"session_meta"/u;
|
|
3402
|
+
var TURN_CONTEXT_LINE_PATTERN = /"type"\s*:\s*"turn_context"/u;
|
|
3403
|
+
var EVENT_MSG_LINE_PATTERN = /"type"\s*:\s*"event_msg"/u;
|
|
3176
3404
|
var TOKEN_COUNT_LINE_PATTERN = /"type"\s*:\s*"token_count"/u;
|
|
3177
3405
|
function shouldParseCodexJsonlLine(lineText) {
|
|
3178
3406
|
if (SESSION_META_LINE_PATTERN.test(lineText) || TURN_CONTEXT_LINE_PATTERN.test(lineText)) {
|
|
@@ -3221,10 +3449,22 @@ function addUsage(left, right) {
|
|
|
3221
3449
|
function hasUsageSignal(usage) {
|
|
3222
3450
|
return usage.inputTokens > 0 || usage.cacheReadTokens > 0 || usage.outputTokens > 0 || usage.reasoningTokens > 0 || usage.totalTokens > 0;
|
|
3223
3451
|
}
|
|
3452
|
+
function hasUsageRollback(current, previous) {
|
|
3453
|
+
return current.inputTokens < previous.inputTokens || current.cacheReadTokens < previous.cacheReadTokens || current.outputTokens < previous.outputTokens || current.reasoningTokens < previous.reasoningTokens || current.totalTokens < previous.totalTokens;
|
|
3454
|
+
}
|
|
3224
3455
|
function deriveDeltaUsage(info, previousTotalUsage) {
|
|
3225
3456
|
const totalUsage = toUsage(info.total_token_usage);
|
|
3226
3457
|
const lastUsage = toUsage(info.last_token_usage);
|
|
3227
3458
|
if (totalUsage && previousTotalUsage) {
|
|
3459
|
+
if (hasUsageRollback(totalUsage, previousTotalUsage)) {
|
|
3460
|
+
if (lastUsage && hasUsageSignal(lastUsage)) {
|
|
3461
|
+
return { deltaUsage: lastUsage, latestTotalUsage: totalUsage };
|
|
3462
|
+
}
|
|
3463
|
+
return {
|
|
3464
|
+
deltaUsage: hasUsageSignal(totalUsage) ? totalUsage : void 0,
|
|
3465
|
+
latestTotalUsage: totalUsage
|
|
3466
|
+
};
|
|
3467
|
+
}
|
|
3228
3468
|
const deltaFromTotals = subtractUsage(totalUsage, previousTotalUsage);
|
|
3229
3469
|
if (hasUsageSignal(deltaFromTotals)) {
|
|
3230
3470
|
return { deltaUsage: deltaFromTotals, latestTotalUsage: totalUsage };
|
|
@@ -3232,7 +3472,7 @@ function deriveDeltaUsage(info, previousTotalUsage) {
|
|
|
3232
3472
|
return { latestTotalUsage: totalUsage };
|
|
3233
3473
|
}
|
|
3234
3474
|
if (lastUsage) {
|
|
3235
|
-
return { deltaUsage: lastUsage, latestTotalUsage: totalUsage };
|
|
3475
|
+
return { deltaUsage: lastUsage, latestTotalUsage: totalUsage, fromLastUsageOnly: true };
|
|
3236
3476
|
}
|
|
3237
3477
|
if (!totalUsage) {
|
|
3238
3478
|
return {};
|
|
@@ -3240,6 +3480,16 @@ function deriveDeltaUsage(info, previousTotalUsage) {
|
|
|
3240
3480
|
const deltaUsage = previousTotalUsage ? subtractUsage(totalUsage, previousTotalUsage) : totalUsage;
|
|
3241
3481
|
return { deltaUsage, latestTotalUsage: totalUsage };
|
|
3242
3482
|
}
|
|
3483
|
+
function createLastUsageOnlyKey(timestamp, usage) {
|
|
3484
|
+
return [
|
|
3485
|
+
timestamp,
|
|
3486
|
+
usage.inputTokens,
|
|
3487
|
+
usage.cacheReadTokens,
|
|
3488
|
+
usage.outputTokens,
|
|
3489
|
+
usage.reasoningTokens,
|
|
3490
|
+
usage.totalTokens
|
|
3491
|
+
].join(":");
|
|
3492
|
+
}
|
|
3243
3493
|
function getFallbackSessionId(filePath) {
|
|
3244
3494
|
return path6.basename(filePath, ".jsonl");
|
|
3245
3495
|
}
|
|
@@ -3251,6 +3501,9 @@ function resolveRepoRootFromPayload(payload) {
|
|
|
3251
3501
|
}
|
|
3252
3502
|
var CodexSourceAdapter = class {
|
|
3253
3503
|
id = "codex";
|
|
3504
|
+
capabilities = {
|
|
3505
|
+
fixedProviderRoots: ["openai"]
|
|
3506
|
+
};
|
|
3254
3507
|
sessionsDir;
|
|
3255
3508
|
requireSessionsDir;
|
|
3256
3509
|
constructor(options = {}) {
|
|
@@ -3308,16 +3561,33 @@ var CodexSourceAdapter = class {
|
|
|
3308
3561
|
if (!info) {
|
|
3309
3562
|
continue;
|
|
3310
3563
|
}
|
|
3311
|
-
const { deltaUsage, latestTotalUsage } = deriveDeltaUsage(
|
|
3564
|
+
const { deltaUsage, latestTotalUsage, fromLastUsageOnly } = deriveDeltaUsage(
|
|
3565
|
+
info,
|
|
3566
|
+
state.previousTotalUsage
|
|
3567
|
+
);
|
|
3312
3568
|
if (!deltaUsage || !hasUsageSignal(deltaUsage)) {
|
|
3569
|
+
if (!fromLastUsageOnly) {
|
|
3570
|
+
state.previousLastUsageOnlyKey = void 0;
|
|
3571
|
+
}
|
|
3313
3572
|
state.previousTotalUsage = latestTotalUsage ?? state.previousTotalUsage;
|
|
3314
3573
|
continue;
|
|
3315
3574
|
}
|
|
3316
|
-
const timestamp =
|
|
3575
|
+
const timestamp = normalizeTimestampCandidate(line.timestamp);
|
|
3317
3576
|
if (!timestamp) {
|
|
3318
|
-
|
|
3577
|
+
if (!fromLastUsageOnly) {
|
|
3578
|
+
state.previousLastUsageOnlyKey = void 0;
|
|
3579
|
+
}
|
|
3319
3580
|
continue;
|
|
3320
3581
|
}
|
|
3582
|
+
if (fromLastUsageOnly) {
|
|
3583
|
+
const currentLastUsageOnlyKey = createLastUsageOnlyKey(timestamp, deltaUsage);
|
|
3584
|
+
if (state.previousLastUsageOnlyKey === currentLastUsageOnlyKey) {
|
|
3585
|
+
continue;
|
|
3586
|
+
}
|
|
3587
|
+
state.previousLastUsageOnlyKey = currentLastUsageOnlyKey;
|
|
3588
|
+
} else {
|
|
3589
|
+
state.previousLastUsageOnlyKey = void 0;
|
|
3590
|
+
}
|
|
3321
3591
|
const model = state.model ?? LEGACY_CODEX_MODEL_FALLBACK;
|
|
3322
3592
|
try {
|
|
3323
3593
|
events.push(
|
|
@@ -3379,24 +3649,6 @@ var DROID_MESSAGE_LINE_PATTERN = /"type"\s*:\s*"message"/u;
|
|
|
3379
3649
|
function shouldParseDroidJsonlLine(lineText) {
|
|
3380
3650
|
return DROID_SESSION_START_LINE_PATTERN.test(lineText) || DROID_MESSAGE_LINE_PATTERN.test(lineText);
|
|
3381
3651
|
}
|
|
3382
|
-
var UNIX_SECONDS_ABS_CUTOFF = 1e10;
|
|
3383
|
-
function normalizeTimestampCandidate(candidate) {
|
|
3384
|
-
let date;
|
|
3385
|
-
if (typeof candidate === "number" && Number.isFinite(candidate)) {
|
|
3386
|
-
const timestampMs = Math.abs(candidate) <= UNIX_SECONDS_ABS_CUTOFF ? candidate * 1e3 : candidate;
|
|
3387
|
-
date = new Date(timestampMs);
|
|
3388
|
-
} else {
|
|
3389
|
-
const normalizedText = asTrimmedText(candidate);
|
|
3390
|
-
if (!normalizedText) {
|
|
3391
|
-
return void 0;
|
|
3392
|
-
}
|
|
3393
|
-
date = new Date(normalizedText);
|
|
3394
|
-
}
|
|
3395
|
-
if (Number.isNaN(date.getTime())) {
|
|
3396
|
-
return void 0;
|
|
3397
|
-
}
|
|
3398
|
-
return date.toISOString();
|
|
3399
|
-
}
|
|
3400
3652
|
function getSettingsSessionId(filePath) {
|
|
3401
3653
|
return path7.basename(filePath, ".settings.json");
|
|
3402
3654
|
}
|
|
@@ -3437,6 +3689,9 @@ var DroidSourceAdapter = class {
|
|
|
3437
3689
|
}
|
|
3438
3690
|
return discoverFiles(normalizedDir, { extension: ".settings.json" });
|
|
3439
3691
|
}
|
|
3692
|
+
async getParseDependencies(filePath) {
|
|
3693
|
+
return [getSiblingJsonlPath(filePath)];
|
|
3694
|
+
}
|
|
3440
3695
|
async parseFile(filePath) {
|
|
3441
3696
|
const { events } = await this.parseFileWithDiagnostics(filePath);
|
|
3442
3697
|
return events;
|
|
@@ -3474,7 +3729,8 @@ var DroidSourceAdapter = class {
|
|
|
3474
3729
|
toNumberLike(tokenUsage.cacheCreationTokens)
|
|
3475
3730
|
);
|
|
3476
3731
|
const billableTokens = inputTokens + outputTokens + cacheReadTokens + cacheWriteTokens;
|
|
3477
|
-
|
|
3732
|
+
const hasUsageSignal3 = billableTokens > 0 || reasoningTokens > 0;
|
|
3733
|
+
if (!hasUsageSignal3) {
|
|
3478
3734
|
skippedRows++;
|
|
3479
3735
|
incrementSkippedReason(skippedRowReasons, "no_token_usage");
|
|
3480
3736
|
return toParseDiagnostics(events, skippedRows, skippedRowReasons);
|
|
@@ -3540,10 +3796,13 @@ var DroidSourceAdapter = class {
|
|
|
3540
3796
|
};
|
|
3541
3797
|
|
|
3542
3798
|
// src/sources/gemini/gemini-source-adapter.ts
|
|
3543
|
-
import { readFile as readFile3 } from "fs/promises";
|
|
3799
|
+
import { readdir as readdir2, readFile as readFile3 } from "fs/promises";
|
|
3544
3800
|
import os4 from "os";
|
|
3545
3801
|
import path8 from "path";
|
|
3546
3802
|
var defaultGeminiDir = path8.join(os4.homedir(), ".gemini");
|
|
3803
|
+
function getProjectsJsonPath(geminiDir) {
|
|
3804
|
+
return path8.join(geminiDir, "projects.json");
|
|
3805
|
+
}
|
|
3547
3806
|
function parseProjectsJson(data) {
|
|
3548
3807
|
const mapping = /* @__PURE__ */ new Map();
|
|
3549
3808
|
const record = asRecord(data);
|
|
@@ -3564,26 +3823,41 @@ function parseProjectsJson(data) {
|
|
|
3564
3823
|
return mapping;
|
|
3565
3824
|
}
|
|
3566
3825
|
async function loadProjectsJson(geminiDir) {
|
|
3567
|
-
const projectsPath =
|
|
3826
|
+
const projectsPath = getProjectsJsonPath(geminiDir);
|
|
3568
3827
|
try {
|
|
3569
3828
|
const content = await readFile3(projectsPath, "utf8");
|
|
3570
3829
|
const parsed = JSON.parse(content);
|
|
3571
3830
|
return parseProjectsJson(parsed);
|
|
3572
|
-
} catch {
|
|
3573
|
-
|
|
3831
|
+
} catch (error) {
|
|
3832
|
+
if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") {
|
|
3833
|
+
return /* @__PURE__ */ new Map();
|
|
3834
|
+
}
|
|
3835
|
+
throw error;
|
|
3574
3836
|
}
|
|
3575
3837
|
}
|
|
3576
3838
|
async function discoverSessionFiles(geminiDir) {
|
|
3577
3839
|
const tmpDir = path8.join(geminiDir, "tmp");
|
|
3840
|
+
let projectEntries;
|
|
3841
|
+
try {
|
|
3842
|
+
projectEntries = await readdir2(tmpDir, { withFileTypes: true, encoding: "utf8" });
|
|
3843
|
+
} catch (error) {
|
|
3844
|
+
if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") {
|
|
3845
|
+
return [];
|
|
3846
|
+
}
|
|
3847
|
+
throw error;
|
|
3848
|
+
}
|
|
3578
3849
|
const allSessionFiles = [];
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
if (
|
|
3583
|
-
|
|
3850
|
+
for (const projectEntry of projectEntries.sort(
|
|
3851
|
+
(left, right) => compareByCodePoint(left.name, right.name)
|
|
3852
|
+
)) {
|
|
3853
|
+
if (!projectEntry.isDirectory() && !projectEntry.isSymbolicLink()) {
|
|
3854
|
+
continue;
|
|
3584
3855
|
}
|
|
3856
|
+
const chatsDir = path8.join(tmpDir, projectEntry.name, "chats");
|
|
3857
|
+
const discoveredChatFiles = await discoverFiles(chatsDir, { extension: ".json" });
|
|
3858
|
+
allSessionFiles.push(...discoveredChatFiles);
|
|
3585
3859
|
}
|
|
3586
|
-
return allSessionFiles;
|
|
3860
|
+
return allSessionFiles.sort(compareByCodePoint);
|
|
3587
3861
|
}
|
|
3588
3862
|
function resolveRepoRoot(filePath, sessionData, projectMapping) {
|
|
3589
3863
|
const projectHash = asTrimmedText(sessionData.projectHash);
|
|
@@ -3642,21 +3916,13 @@ function extractTokenUsage(tokens) {
|
|
|
3642
3916
|
totalTokens
|
|
3643
3917
|
};
|
|
3644
3918
|
}
|
|
3645
|
-
function normalizeTimestamp2(candidate) {
|
|
3646
|
-
if (typeof candidate !== "string" || isBlankText(candidate)) {
|
|
3647
|
-
return void 0;
|
|
3648
|
-
}
|
|
3649
|
-
const date = new Date(candidate.trim());
|
|
3650
|
-
if (Number.isNaN(date.getTime())) {
|
|
3651
|
-
return void 0;
|
|
3652
|
-
}
|
|
3653
|
-
return date.toISOString();
|
|
3654
|
-
}
|
|
3655
3919
|
var GeminiSourceAdapter = class {
|
|
3656
3920
|
id = "gemini";
|
|
3921
|
+
capabilities = {
|
|
3922
|
+
fixedProviderRoots: ["google"]
|
|
3923
|
+
};
|
|
3657
3924
|
geminiDir;
|
|
3658
3925
|
requireGeminiDir;
|
|
3659
|
-
projectMapping = null;
|
|
3660
3926
|
constructor(options = {}) {
|
|
3661
3927
|
this.geminiDir = options.geminiDir ?? defaultGeminiDir;
|
|
3662
3928
|
this.requireGeminiDir = options.requireGeminiDir ?? false;
|
|
@@ -3667,22 +3933,11 @@ var GeminiSourceAdapter = class {
|
|
|
3667
3933
|
}
|
|
3668
3934
|
return this.geminiDir.trim();
|
|
3669
3935
|
}
|
|
3670
|
-
async getProjectMapping(normalizedGeminiDir) {
|
|
3671
|
-
if (this.projectMapping) {
|
|
3672
|
-
return this.projectMapping;
|
|
3673
|
-
}
|
|
3674
|
-
this.projectMapping = await loadProjectsJson(normalizedGeminiDir);
|
|
3675
|
-
return this.projectMapping;
|
|
3676
|
-
}
|
|
3677
3936
|
async getProjectMappingForParse() {
|
|
3678
|
-
if (this.projectMapping) {
|
|
3679
|
-
return this.projectMapping;
|
|
3680
|
-
}
|
|
3681
3937
|
if (isBlankText(this.geminiDir)) {
|
|
3682
3938
|
return /* @__PURE__ */ new Map();
|
|
3683
3939
|
}
|
|
3684
|
-
|
|
3685
|
-
return this.projectMapping;
|
|
3940
|
+
return loadProjectsJson(this.geminiDir.trim());
|
|
3686
3941
|
}
|
|
3687
3942
|
async discoverFiles() {
|
|
3688
3943
|
const normalizedDir = this.getNormalizedGeminiDir();
|
|
@@ -3692,9 +3947,14 @@ var GeminiSourceAdapter = class {
|
|
|
3692
3947
|
if (this.requireGeminiDir && !await pathIsDirectory(normalizedDir)) {
|
|
3693
3948
|
throw new Error(`Gemini directory is not a directory: ${normalizedDir}`);
|
|
3694
3949
|
}
|
|
3695
|
-
await this.getProjectMapping(normalizedDir);
|
|
3696
3950
|
return discoverSessionFiles(normalizedDir);
|
|
3697
3951
|
}
|
|
3952
|
+
async getParseDependencies() {
|
|
3953
|
+
if (isBlankText(this.geminiDir)) {
|
|
3954
|
+
return [];
|
|
3955
|
+
}
|
|
3956
|
+
return [getProjectsJsonPath(this.getNormalizedGeminiDir())];
|
|
3957
|
+
}
|
|
3698
3958
|
async parseFile(filePath) {
|
|
3699
3959
|
const { events } = await this.parseFileWithDiagnostics(filePath);
|
|
3700
3960
|
return events;
|
|
@@ -3745,7 +4005,7 @@ var GeminiSourceAdapter = class {
|
|
|
3745
4005
|
incrementSkippedReason(skippedRowReasons, "no_token_usage");
|
|
3746
4006
|
continue;
|
|
3747
4007
|
}
|
|
3748
|
-
const timestamp =
|
|
4008
|
+
const timestamp = normalizeTimestampCandidate(message.timestamp);
|
|
3749
4009
|
if (!timestamp) {
|
|
3750
4010
|
skippedRows++;
|
|
3751
4011
|
incrementSkippedReason(skippedRowReasons, "invalid_timestamp");
|
|
@@ -3780,8 +4040,15 @@ import path9 from "path";
|
|
|
3780
4040
|
function deduplicate(paths) {
|
|
3781
4041
|
return [...new Set(paths)];
|
|
3782
4042
|
}
|
|
4043
|
+
function normalizeEnvPath(value) {
|
|
4044
|
+
if (value === void 0) {
|
|
4045
|
+
return void 0;
|
|
4046
|
+
}
|
|
4047
|
+
const normalized = value.trim();
|
|
4048
|
+
return normalized || void 0;
|
|
4049
|
+
}
|
|
3783
4050
|
function getLinuxLikeCandidates(homeDir, env) {
|
|
3784
|
-
const xdgDataHome = env.XDG_DATA_HOME ?? path9.join(homeDir, ".local", "share");
|
|
4051
|
+
const xdgDataHome = normalizeEnvPath(env.XDG_DATA_HOME) ?? path9.join(homeDir, ".local", "share");
|
|
3785
4052
|
return [
|
|
3786
4053
|
path9.join(xdgDataHome, "opencode", "opencode.db"),
|
|
3787
4054
|
path9.join(xdgDataHome, "opencode", "db.sqlite"),
|
|
@@ -3799,7 +4066,8 @@ function getMacOsCandidates(homeDir) {
|
|
|
3799
4066
|
];
|
|
3800
4067
|
}
|
|
3801
4068
|
function getWindowsCandidates(homeDir, env) {
|
|
3802
|
-
const
|
|
4069
|
+
const userProfile = normalizeEnvPath(env.USERPROFILE);
|
|
4070
|
+
const roamingBase = normalizeEnvPath(env.APPDATA) ?? normalizeEnvPath(env.LOCALAPPDATA) ?? (userProfile ? path9.join(userProfile, "AppData", "Roaming") : void 0);
|
|
3803
4071
|
const roamingCandidates = roamingBase ? [
|
|
3804
4072
|
path9.join(roamingBase, "opencode", "opencode.db"),
|
|
3805
4073
|
path9.join(roamingBase, "opencode", "db.sqlite")
|
|
@@ -3878,33 +4146,6 @@ async function loadNodeSqliteModule() {
|
|
|
3878
4146
|
}
|
|
3879
4147
|
|
|
3880
4148
|
// src/sources/opencode/opencode-row-parser.ts
|
|
3881
|
-
var UNIX_SECONDS_ABS_CUTOFF2 = 1e10;
|
|
3882
|
-
function normalizeTimestampCandidate2(candidate) {
|
|
3883
|
-
if (typeof candidate === "number" && Number.isFinite(candidate)) {
|
|
3884
|
-
const timestampMs = Math.abs(candidate) <= UNIX_SECONDS_ABS_CUTOFF2 ? candidate * 1e3 : candidate;
|
|
3885
|
-
const date = new Date(timestampMs);
|
|
3886
|
-
if (Number.isNaN(date.getTime())) {
|
|
3887
|
-
return void 0;
|
|
3888
|
-
}
|
|
3889
|
-
return date.toISOString();
|
|
3890
|
-
}
|
|
3891
|
-
if (typeof candidate === "string") {
|
|
3892
|
-
const trimmed = candidate.trim();
|
|
3893
|
-
if (!trimmed) {
|
|
3894
|
-
return void 0;
|
|
3895
|
-
}
|
|
3896
|
-
const numericTimestamp = Number(trimmed);
|
|
3897
|
-
if (Number.isFinite(numericTimestamp)) {
|
|
3898
|
-
return normalizeTimestampCandidate2(numericTimestamp);
|
|
3899
|
-
}
|
|
3900
|
-
const date = new Date(trimmed);
|
|
3901
|
-
if (Number.isNaN(date.getTime())) {
|
|
3902
|
-
return void 0;
|
|
3903
|
-
}
|
|
3904
|
-
return date.toISOString();
|
|
3905
|
-
}
|
|
3906
|
-
return void 0;
|
|
3907
|
-
}
|
|
3908
4149
|
function resolveTimestamp(rowTimestamp, messagePayload) {
|
|
3909
4150
|
const timestampCandidates = [
|
|
3910
4151
|
rowTimestamp,
|
|
@@ -3913,7 +4154,7 @@ function resolveTimestamp(rowTimestamp, messagePayload) {
|
|
|
3913
4154
|
messagePayload.time_created
|
|
3914
4155
|
];
|
|
3915
4156
|
for (const candidate of timestampCandidates) {
|
|
3916
|
-
const resolved =
|
|
4157
|
+
const resolved = normalizeTimestampCandidate(candidate);
|
|
3917
4158
|
if (resolved) {
|
|
3918
4159
|
return resolved;
|
|
3919
4160
|
}
|
|
@@ -4229,6 +4470,9 @@ async function sleep2(delayMs) {
|
|
|
4229
4470
|
setTimeout(resolve, delayMs);
|
|
4230
4471
|
});
|
|
4231
4472
|
}
|
|
4473
|
+
function getOpenCodeParseDependencies(dbPath) {
|
|
4474
|
+
return [`${dbPath}-wal`, `${dbPath}-shm`, `${dbPath}-journal`];
|
|
4475
|
+
}
|
|
4232
4476
|
var OpenCodeSourceAdapter = class {
|
|
4233
4477
|
id = "opencode";
|
|
4234
4478
|
explicitDbPath;
|
|
@@ -4287,6 +4531,12 @@ var OpenCodeSourceAdapter = class {
|
|
|
4287
4531
|
const parseDiagnostics = await this.parseFileWithDiagnostics(dbPath);
|
|
4288
4532
|
return parseDiagnostics.events;
|
|
4289
4533
|
}
|
|
4534
|
+
async getParseDependencies(dbPath) {
|
|
4535
|
+
if (isBlankText2(dbPath)) {
|
|
4536
|
+
return [];
|
|
4537
|
+
}
|
|
4538
|
+
return getOpenCodeParseDependencies(dbPath.trim());
|
|
4539
|
+
}
|
|
4290
4540
|
async parseFileWithDiagnostics(dbPath) {
|
|
4291
4541
|
if (isBlankText2(dbPath)) {
|
|
4292
4542
|
throw new Error("OpenCode DB path must be a non-empty path");
|
|
@@ -4328,29 +4578,10 @@ var PI_MODEL_CHANGE_LINE_PATTERN = /"type"\s*:\s*"model_change"/u;
|
|
|
4328
4578
|
function shouldParsePiJsonlLine(lineText) {
|
|
4329
4579
|
return PI_MESSAGE_LINE_PATTERN.test(lineText) || PI_SESSION_LINE_PATTERN.test(lineText) || PI_MODEL_CHANGE_LINE_PATTERN.test(lineText);
|
|
4330
4580
|
}
|
|
4331
|
-
var allowAllProviders = () => true;
|
|
4332
|
-
var UNIX_SECONDS_ABS_CUTOFF3 = 1e10;
|
|
4333
|
-
function normalizeTimestampCandidate3(candidate) {
|
|
4334
|
-
let date;
|
|
4335
|
-
if (typeof candidate === "number" && Number.isFinite(candidate)) {
|
|
4336
|
-
const timestampMs = Math.abs(candidate) <= UNIX_SECONDS_ABS_CUTOFF3 ? candidate * 1e3 : candidate;
|
|
4337
|
-
date = new Date(timestampMs);
|
|
4338
|
-
} else {
|
|
4339
|
-
const normalizedText = asTrimmedText(candidate);
|
|
4340
|
-
if (!normalizedText) {
|
|
4341
|
-
return void 0;
|
|
4342
|
-
}
|
|
4343
|
-
date = new Date(normalizedText);
|
|
4344
|
-
}
|
|
4345
|
-
if (Number.isNaN(date.getTime())) {
|
|
4346
|
-
return void 0;
|
|
4347
|
-
}
|
|
4348
|
-
return date.toISOString();
|
|
4349
|
-
}
|
|
4350
4581
|
function resolveTimestamp2(line, message, state) {
|
|
4351
4582
|
const candidates = [line.timestamp, message?.timestamp, state.sessionTimestamp];
|
|
4352
4583
|
for (const candidate of candidates) {
|
|
4353
|
-
const normalizedTimestamp =
|
|
4584
|
+
const normalizedTimestamp = normalizeTimestampCandidate(candidate);
|
|
4354
4585
|
if (normalizedTimestamp) {
|
|
4355
4586
|
return normalizedTimestamp;
|
|
4356
4587
|
}
|
|
@@ -4426,11 +4657,9 @@ function resolveRepoRootFromRecord(record) {
|
|
|
4426
4657
|
var PiSourceAdapter = class {
|
|
4427
4658
|
id = "pi";
|
|
4428
4659
|
sessionsDir;
|
|
4429
|
-
providerFilter;
|
|
4430
4660
|
requireSessionsDir;
|
|
4431
4661
|
constructor(options = {}) {
|
|
4432
4662
|
this.sessionsDir = options.sessionsDir ?? defaultSessionsDir3;
|
|
4433
|
-
this.providerFilter = options.providerFilter ?? allowAllProviders;
|
|
4434
4663
|
this.requireSessionsDir = options.requireSessionsDir ?? false;
|
|
4435
4664
|
}
|
|
4436
4665
|
async discoverFiles() {
|
|
@@ -4473,9 +4702,6 @@ var PiSourceAdapter = class {
|
|
|
4473
4702
|
continue;
|
|
4474
4703
|
}
|
|
4475
4704
|
const provider = asTrimmedText(line.provider) ?? asTrimmedText(message?.provider) ?? state.provider;
|
|
4476
|
-
if (!this.providerFilter(provider)) {
|
|
4477
|
-
continue;
|
|
4478
|
-
}
|
|
4479
4705
|
const timestamp = resolveTimestamp2(line, message, state);
|
|
4480
4706
|
if (!timestamp || !state.sessionId) {
|
|
4481
4707
|
continue;
|
|
@@ -4502,7 +4728,7 @@ var PiSourceAdapter = class {
|
|
|
4502
4728
|
}
|
|
4503
4729
|
};
|
|
4504
4730
|
|
|
4505
|
-
// src/
|
|
4731
|
+
// src/utils/source-directory-overrides.ts
|
|
4506
4732
|
function parseSourceDirectoryOverrides(entries) {
|
|
4507
4733
|
const overrides = /* @__PURE__ */ new Map();
|
|
4508
4734
|
if (!entries || entries.length === 0) {
|
|
@@ -4525,6 +4751,8 @@ function parseSourceDirectoryOverrides(entries) {
|
|
|
4525
4751
|
}
|
|
4526
4752
|
return overrides;
|
|
4527
4753
|
}
|
|
4754
|
+
|
|
4755
|
+
// src/sources/create-default-adapters.ts
|
|
4528
4756
|
var sourceRegistrations = [
|
|
4529
4757
|
{
|
|
4530
4758
|
id: "pi",
|
|
@@ -4757,23 +4985,6 @@ function validateBuildOptions(options) {
|
|
|
4757
4985
|
normalizedPricingUrl: validatePricingUrl(options.pricingUrl)
|
|
4758
4986
|
};
|
|
4759
4987
|
}
|
|
4760
|
-
function parseSourceDirOverrideIds(sourceDirEntries) {
|
|
4761
|
-
const overrideIds = /* @__PURE__ */ new Set();
|
|
4762
|
-
if (!sourceDirEntries || sourceDirEntries.length === 0) {
|
|
4763
|
-
return overrideIds;
|
|
4764
|
-
}
|
|
4765
|
-
for (const entry of sourceDirEntries) {
|
|
4766
|
-
const separatorIndex = entry.indexOf("=");
|
|
4767
|
-
if (separatorIndex <= 0) {
|
|
4768
|
-
continue;
|
|
4769
|
-
}
|
|
4770
|
-
const sourceId = entry.slice(0, separatorIndex).trim().toLowerCase();
|
|
4771
|
-
if (sourceId.length > 0) {
|
|
4772
|
-
overrideIds.add(sourceId);
|
|
4773
|
-
}
|
|
4774
|
-
}
|
|
4775
|
-
return overrideIds;
|
|
4776
|
-
}
|
|
4777
4988
|
function resolveExplicitSourceIds(options, sourceFilter) {
|
|
4778
4989
|
const explicitSourceIds = /* @__PURE__ */ new Set();
|
|
4779
4990
|
if (sourceFilter) {
|
|
@@ -4781,7 +4992,7 @@ function resolveExplicitSourceIds(options, sourceFilter) {
|
|
|
4781
4992
|
explicitSourceIds.add(sourceId);
|
|
4782
4993
|
}
|
|
4783
4994
|
}
|
|
4784
|
-
for (const sourceId of
|
|
4995
|
+
for (const sourceId of parseSourceDirectoryOverrides(options.sourceDir).keys()) {
|
|
4785
4996
|
explicitSourceIds.add(sourceId);
|
|
4786
4997
|
}
|
|
4787
4998
|
if (options.piDir) {
|
|
@@ -4818,24 +5029,84 @@ function normalizeBuildUsageInputs(options) {
|
|
|
4818
5029
|
const providerFilter = normalizeProviderFilter(options.provider);
|
|
4819
5030
|
const sourceFilter = normalizeSourceFilter(options.source);
|
|
4820
5031
|
const modelFilter = normalizeModelFilter(options.model);
|
|
5032
|
+
const candidateProviderRoots = intersectCanonicalProviderRoots(
|
|
5033
|
+
resolveExplicitProviderRoots(providerFilter),
|
|
5034
|
+
inferCanonicalProviderRootsFromModels(modelFilter)
|
|
5035
|
+
);
|
|
4821
5036
|
const explicitSourceIds = resolveExplicitSourceIds(options, sourceFilter);
|
|
4822
5037
|
return {
|
|
4823
5038
|
timezone,
|
|
4824
5039
|
providerFilter,
|
|
5040
|
+
candidateProviderRoots,
|
|
4825
5041
|
sourceFilter,
|
|
4826
5042
|
modelFilter,
|
|
4827
5043
|
explicitSourceIds,
|
|
4828
5044
|
pricingUrl: normalizedPricingUrl
|
|
4829
5045
|
};
|
|
4830
5046
|
}
|
|
4831
|
-
function selectAdaptersForParsing(adapters,
|
|
5047
|
+
function selectAdaptersForParsing(adapters, options) {
|
|
4832
5048
|
const availableSourceIds = new Set(adapters.map((adapter) => adapter.id.toLowerCase()));
|
|
4833
|
-
validateSourceFilterValues(sourceFilter, availableSourceIds);
|
|
4834
|
-
|
|
5049
|
+
validateSourceFilterValues(options.sourceFilter, availableSourceIds);
|
|
5050
|
+
const sourceFilter = options.sourceFilter;
|
|
5051
|
+
const selectedBySource = sourceFilter ? adapters.filter((adapter) => sourceFilter.has(adapter.id.toLowerCase())) : adapters;
|
|
5052
|
+
if (!options.candidateProviderRoots) {
|
|
5053
|
+
options.runtimeProfile?.recordSourceSelection({
|
|
5054
|
+
availableSourceIds: adapters.map((adapter) => adapter.id),
|
|
5055
|
+
selectedSourceIds: selectedBySource.map((adapter) => adapter.id)
|
|
5056
|
+
});
|
|
5057
|
+
return selectedBySource;
|
|
5058
|
+
}
|
|
5059
|
+
const candidateProviderRootSet = new Set(options.candidateProviderRoots);
|
|
5060
|
+
const selectedAdapters = selectedBySource.filter((adapter) => {
|
|
5061
|
+
const fixedProviderRoots = adapter.capabilities?.fixedProviderRoots;
|
|
5062
|
+
if (!fixedProviderRoots || fixedProviderRoots.length === 0) {
|
|
5063
|
+
return true;
|
|
5064
|
+
}
|
|
5065
|
+
return fixedProviderRoots.some((providerRoot) => candidateProviderRootSet.has(providerRoot));
|
|
5066
|
+
});
|
|
5067
|
+
options.runtimeProfile?.recordSourceSelection({
|
|
5068
|
+
availableSourceIds: adapters.map((adapter) => adapter.id),
|
|
5069
|
+
selectedSourceIds: selectedAdapters.map((adapter) => adapter.id),
|
|
5070
|
+
candidateProviderRoots: options.candidateProviderRoots
|
|
5071
|
+
});
|
|
5072
|
+
return selectedAdapters;
|
|
5073
|
+
}
|
|
5074
|
+
function describeExplicitScopeConstraint(options) {
|
|
5075
|
+
const activeFlags = [];
|
|
5076
|
+
if (options.providerFilter) {
|
|
5077
|
+
activeFlags.push("--provider");
|
|
5078
|
+
}
|
|
5079
|
+
if (options.modelFilter && options.modelFilter.length > 0) {
|
|
5080
|
+
activeFlags.push("--model");
|
|
5081
|
+
}
|
|
5082
|
+
if (activeFlags.length === 0) {
|
|
5083
|
+
return "provider/model";
|
|
5084
|
+
}
|
|
5085
|
+
return activeFlags.join("/");
|
|
5086
|
+
}
|
|
5087
|
+
function throwOnExplicitSourceScopeConflicts(adapters, selectedAdapters, options) {
|
|
5088
|
+
if (!options.candidateProviderRoots || options.explicitSourceIds.size === 0) {
|
|
5089
|
+
return;
|
|
5090
|
+
}
|
|
5091
|
+
const selectedSourceIds = new Set(selectedAdapters.map((adapter) => adapter.id.toLowerCase()));
|
|
5092
|
+
const incompatibleExplicitSources = adapters.filter((adapter) => {
|
|
5093
|
+
const sourceId = adapter.id.toLowerCase();
|
|
5094
|
+
if (!options.explicitSourceIds.has(sourceId) || selectedSourceIds.has(sourceId)) {
|
|
5095
|
+
return false;
|
|
5096
|
+
}
|
|
5097
|
+
const fixedProviderRoots = adapter.capabilities?.fixedProviderRoots;
|
|
5098
|
+
return Boolean(fixedProviderRoots && fixedProviderRoots.length > 0);
|
|
5099
|
+
}).map((adapter) => adapter.id).sort(compareByCodePoint);
|
|
5100
|
+
if (incompatibleExplicitSources.length === 0) {
|
|
5101
|
+
return;
|
|
5102
|
+
}
|
|
5103
|
+
throw new Error(
|
|
5104
|
+
`Explicitly requested source(s) are incompatible with the requested ${describeExplicitScopeConstraint(options)} scope: ${incompatibleExplicitSources.join(", ")}.`
|
|
5105
|
+
);
|
|
4835
5106
|
}
|
|
4836
5107
|
|
|
4837
5108
|
// src/cli/build-usage-data-parsing.ts
|
|
4838
|
-
import { stat as
|
|
5109
|
+
import { stat as stat5 } from "fs/promises";
|
|
4839
5110
|
|
|
4840
5111
|
// src/cli/normalize-skipped-row-reasons.ts
|
|
4841
5112
|
function toPositiveInteger(value) {
|
|
@@ -4863,9 +5134,9 @@ function normalizeSkippedRowReasons(value) {
|
|
|
4863
5134
|
}
|
|
4864
5135
|
|
|
4865
5136
|
// src/cli/parse-file-cache.ts
|
|
4866
|
-
import { mkdir as mkdir2, readFile as readFile4, rename, rm, stat as
|
|
5137
|
+
import { mkdir as mkdir2, readFile as readFile4, rename, rm, stat as stat4, writeFile as writeFile2 } from "fs/promises";
|
|
4867
5138
|
import path11 from "path";
|
|
4868
|
-
var PARSE_FILE_CACHE_VERSION =
|
|
5139
|
+
var PARSE_FILE_CACHE_VERSION = 5;
|
|
4869
5140
|
var CACHE_KEY_SEPARATOR = "\0";
|
|
4870
5141
|
function createCacheKey(source, filePath) {
|
|
4871
5142
|
return `${source}${CACHE_KEY_SEPARATOR}${filePath}`;
|
|
@@ -4956,7 +5227,7 @@ function cloneUsageEvents(events) {
|
|
|
4956
5227
|
return events.map((event) => cloneUsageEvent(event));
|
|
4957
5228
|
}
|
|
4958
5229
|
function cloneSkippedRowReasons(skippedRowReasons) {
|
|
4959
|
-
return (skippedRowReasons ?? []).map((
|
|
5230
|
+
return (skippedRowReasons ?? []).map((stat6) => ({ reason: stat6.reason, count: stat6.count }));
|
|
4960
5231
|
}
|
|
4961
5232
|
function normalizeCachedEvents(value) {
|
|
4962
5233
|
if (!Array.isArray(value)) {
|
|
@@ -4972,6 +5243,94 @@ function normalizeCachedEvents(value) {
|
|
|
4972
5243
|
}
|
|
4973
5244
|
return normalizedEvents;
|
|
4974
5245
|
}
|
|
5246
|
+
function normalizeFingerprintPath(value) {
|
|
5247
|
+
if (typeof value !== "string") {
|
|
5248
|
+
return void 0;
|
|
5249
|
+
}
|
|
5250
|
+
const normalized = value.trim();
|
|
5251
|
+
return normalized || void 0;
|
|
5252
|
+
}
|
|
5253
|
+
function normalizeParseDependencyFingerprint(value) {
|
|
5254
|
+
const record = asRecord(value);
|
|
5255
|
+
if (!record) {
|
|
5256
|
+
return void 0;
|
|
5257
|
+
}
|
|
5258
|
+
const dependencyPath = normalizeFingerprintPath(record.path);
|
|
5259
|
+
const exists = typeof record.exists === "boolean" ? record.exists : void 0;
|
|
5260
|
+
if (!dependencyPath || exists === void 0) {
|
|
5261
|
+
return void 0;
|
|
5262
|
+
}
|
|
5263
|
+
if (!exists) {
|
|
5264
|
+
return {
|
|
5265
|
+
path: dependencyPath,
|
|
5266
|
+
exists: false
|
|
5267
|
+
};
|
|
5268
|
+
}
|
|
5269
|
+
const size = toNonNegativeInteger(record.size);
|
|
5270
|
+
const mtimeMs = toNonNegativeNumber2(record.mtimeMs);
|
|
5271
|
+
if (size === void 0 || mtimeMs === void 0) {
|
|
5272
|
+
return void 0;
|
|
5273
|
+
}
|
|
5274
|
+
return {
|
|
5275
|
+
path: dependencyPath,
|
|
5276
|
+
exists: true,
|
|
5277
|
+
size,
|
|
5278
|
+
mtimeMs
|
|
5279
|
+
};
|
|
5280
|
+
}
|
|
5281
|
+
function compareDependencyFingerprint(left, right) {
|
|
5282
|
+
if (left.path !== right.path) {
|
|
5283
|
+
return compareByCodePoint(left.path, right.path);
|
|
5284
|
+
}
|
|
5285
|
+
if (left.exists !== right.exists) {
|
|
5286
|
+
return left.exists ? 1 : -1;
|
|
5287
|
+
}
|
|
5288
|
+
if ((left.size ?? -1) !== (right.size ?? -1)) {
|
|
5289
|
+
return (left.size ?? -1) - (right.size ?? -1);
|
|
5290
|
+
}
|
|
5291
|
+
return (left.mtimeMs ?? -1) - (right.mtimeMs ?? -1);
|
|
5292
|
+
}
|
|
5293
|
+
function normalizeRuntimeFingerprint(filePath, fingerprint) {
|
|
5294
|
+
const fingerprintRecord = asRecord(fingerprint);
|
|
5295
|
+
if (!fingerprintRecord) {
|
|
5296
|
+
return void 0;
|
|
5297
|
+
}
|
|
5298
|
+
if ("dependencies" in fingerprintRecord && !Array.isArray(fingerprintRecord.dependencies)) {
|
|
5299
|
+
return void 0;
|
|
5300
|
+
}
|
|
5301
|
+
if (Array.isArray(fingerprintRecord.dependencies)) {
|
|
5302
|
+
const normalizedDependencies = [];
|
|
5303
|
+
for (const dependency of fingerprintRecord.dependencies) {
|
|
5304
|
+
const normalizedDependency = normalizeParseDependencyFingerprint(dependency);
|
|
5305
|
+
if (!normalizedDependency) {
|
|
5306
|
+
return void 0;
|
|
5307
|
+
}
|
|
5308
|
+
normalizedDependencies.push(normalizedDependency);
|
|
5309
|
+
}
|
|
5310
|
+
if (normalizedDependencies.length === 0) {
|
|
5311
|
+
return void 0;
|
|
5312
|
+
}
|
|
5313
|
+
normalizedDependencies.sort(compareDependencyFingerprint);
|
|
5314
|
+
return {
|
|
5315
|
+
dependencies: normalizedDependencies
|
|
5316
|
+
};
|
|
5317
|
+
}
|
|
5318
|
+
const size = toNonNegativeInteger(fingerprintRecord.size);
|
|
5319
|
+
const mtimeMs = toNonNegativeNumber2(fingerprintRecord.mtimeMs);
|
|
5320
|
+
if (size === void 0 || mtimeMs === void 0) {
|
|
5321
|
+
return void 0;
|
|
5322
|
+
}
|
|
5323
|
+
return {
|
|
5324
|
+
dependencies: [
|
|
5325
|
+
{
|
|
5326
|
+
path: filePath,
|
|
5327
|
+
exists: true,
|
|
5328
|
+
size,
|
|
5329
|
+
mtimeMs
|
|
5330
|
+
}
|
|
5331
|
+
]
|
|
5332
|
+
};
|
|
5333
|
+
}
|
|
4975
5334
|
function normalizeCacheEntry(value) {
|
|
4976
5335
|
const record = asRecord(value);
|
|
4977
5336
|
if (!record) {
|
|
@@ -4980,13 +5339,11 @@ function normalizeCacheEntry(value) {
|
|
|
4980
5339
|
const source = normalizeSourceId(record.source)?.toLowerCase() ?? "";
|
|
4981
5340
|
const filePath = typeof record.filePath === "string" ? record.filePath.trim() : "";
|
|
4982
5341
|
const cachedAt = toNonNegativeInteger(record.cachedAt);
|
|
4983
|
-
const fingerprint =
|
|
5342
|
+
const fingerprint = normalizeRuntimeFingerprint(filePath, record.fingerprint);
|
|
4984
5343
|
const diagnostics = asRecord(record.diagnostics);
|
|
4985
|
-
const size = toNonNegativeInteger(fingerprint?.size);
|
|
4986
|
-
const mtimeMs = toNonNegativeNumber2(fingerprint?.mtimeMs);
|
|
4987
5344
|
const skippedRows = toNonNegativeInteger(diagnostics?.skippedRows) ?? 0;
|
|
4988
5345
|
const events = normalizeCachedEvents(diagnostics?.events);
|
|
4989
|
-
if (!source || !filePath ||
|
|
5346
|
+
if (!source || !filePath || !fingerprint || cachedAt === void 0) {
|
|
4990
5347
|
return void 0;
|
|
4991
5348
|
}
|
|
4992
5349
|
if (!events) {
|
|
@@ -4995,7 +5352,7 @@ function normalizeCacheEntry(value) {
|
|
|
4995
5352
|
return {
|
|
4996
5353
|
source,
|
|
4997
5354
|
filePath,
|
|
4998
|
-
fingerprint
|
|
5355
|
+
fingerprint,
|
|
4999
5356
|
cachedAt,
|
|
5000
5357
|
diagnostics: {
|
|
5001
5358
|
events,
|
|
@@ -5051,7 +5408,14 @@ var ParseFileCache = class _ParseFileCache {
|
|
|
5051
5408
|
this.dirty = true;
|
|
5052
5409
|
return void 0;
|
|
5053
5410
|
}
|
|
5054
|
-
|
|
5411
|
+
const normalizedFingerprint = normalizeRuntimeFingerprint(filePath, fingerprint);
|
|
5412
|
+
if (!normalizedFingerprint) {
|
|
5413
|
+
return void 0;
|
|
5414
|
+
}
|
|
5415
|
+
if (entry.fingerprint.dependencies.length !== normalizedFingerprint.dependencies.length || entry.fingerprint.dependencies.some((dependency, index) => {
|
|
5416
|
+
const candidate = normalizedFingerprint.dependencies[index];
|
|
5417
|
+
return dependency.path !== candidate.path || dependency.exists !== candidate.exists || dependency.size !== candidate.size || dependency.mtimeMs !== candidate.mtimeMs;
|
|
5418
|
+
})) {
|
|
5055
5419
|
this.entriesByKey.delete(cacheKey);
|
|
5056
5420
|
this.dirty = true;
|
|
5057
5421
|
return void 0;
|
|
@@ -5064,13 +5428,14 @@ var ParseFileCache = class _ParseFileCache {
|
|
|
5064
5428
|
}
|
|
5065
5429
|
set(source, filePath, fingerprint, diagnostics) {
|
|
5066
5430
|
const normalizedSource = normalizeCacheSource(source);
|
|
5431
|
+
const normalizedFingerprint = normalizeRuntimeFingerprint(filePath, fingerprint);
|
|
5432
|
+
if (!normalizedFingerprint) {
|
|
5433
|
+
return;
|
|
5434
|
+
}
|
|
5067
5435
|
this.entriesByKey.set(createCacheKey(normalizedSource, filePath), {
|
|
5068
5436
|
source: normalizedSource,
|
|
5069
5437
|
filePath,
|
|
5070
|
-
fingerprint:
|
|
5071
|
-
size: fingerprint.size,
|
|
5072
|
-
mtimeMs: fingerprint.mtimeMs
|
|
5073
|
-
},
|
|
5438
|
+
fingerprint: normalizedFingerprint,
|
|
5074
5439
|
cachedAt: this.now(),
|
|
5075
5440
|
diagnostics: {
|
|
5076
5441
|
events: cloneUsageEvents(diagnostics.events),
|
|
@@ -5111,6 +5476,7 @@ var ParseFileCache = class _ParseFileCache {
|
|
|
5111
5476
|
try {
|
|
5112
5477
|
await writeFile2(temporaryPath, payloadText, "utf8");
|
|
5113
5478
|
await rename(temporaryPath, this.cacheFilePath);
|
|
5479
|
+
this.replaceEntries(keptEntries);
|
|
5114
5480
|
this.dirty = false;
|
|
5115
5481
|
} catch (error) {
|
|
5116
5482
|
await rm(temporaryPath, { force: true }).catch(() => void 0);
|
|
@@ -5133,10 +5499,16 @@ var ParseFileCache = class _ParseFileCache {
|
|
|
5133
5499
|
}))
|
|
5134
5500
|
};
|
|
5135
5501
|
}
|
|
5502
|
+
replaceEntries(entries) {
|
|
5503
|
+
this.entriesByKey.clear();
|
|
5504
|
+
for (const entry of entries) {
|
|
5505
|
+
this.entriesByKey.set(createCacheKey(entry.source, entry.filePath), entry);
|
|
5506
|
+
}
|
|
5507
|
+
}
|
|
5136
5508
|
async loadFromDisk() {
|
|
5137
5509
|
let cacheFileSizeBytes;
|
|
5138
5510
|
try {
|
|
5139
|
-
const cacheStat = await
|
|
5511
|
+
const cacheStat = await stat4(this.cacheFilePath);
|
|
5140
5512
|
cacheFileSizeBytes = cacheStat.size;
|
|
5141
5513
|
} catch {
|
|
5142
5514
|
return;
|
|
@@ -5189,6 +5561,8 @@ var ParseFileCache = class _ParseFileCache {
|
|
|
5189
5561
|
);
|
|
5190
5562
|
}
|
|
5191
5563
|
if (this.entriesByKey.size > this.limits.maxEntries) {
|
|
5564
|
+
const keptEntries = [...this.entriesByKey.values()].sort((left, right) => right.cachedAt - left.cachedAt).slice(0, this.limits.maxEntries);
|
|
5565
|
+
this.replaceEntries(keptEntries);
|
|
5192
5566
|
this.dirty = true;
|
|
5193
5567
|
}
|
|
5194
5568
|
}
|
|
@@ -5204,7 +5578,82 @@ function normalizeSkippedRowsCount(value) {
|
|
|
5204
5578
|
}
|
|
5205
5579
|
return Math.max(0, Math.trunc(value));
|
|
5206
5580
|
}
|
|
5207
|
-
|
|
5581
|
+
function isMissingPathError(error) {
|
|
5582
|
+
return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
|
|
5583
|
+
}
|
|
5584
|
+
async function createParseDependencyFingerprint(filePath, options) {
|
|
5585
|
+
try {
|
|
5586
|
+
const fileStat = await stat5(filePath);
|
|
5587
|
+
return {
|
|
5588
|
+
path: filePath,
|
|
5589
|
+
exists: true,
|
|
5590
|
+
size: fileStat.size,
|
|
5591
|
+
mtimeMs: fileStat.mtimeMs
|
|
5592
|
+
};
|
|
5593
|
+
} catch (error) {
|
|
5594
|
+
if (options.allowMissing && isMissingPathError(error)) {
|
|
5595
|
+
return {
|
|
5596
|
+
path: filePath,
|
|
5597
|
+
exists: false
|
|
5598
|
+
};
|
|
5599
|
+
}
|
|
5600
|
+
return void 0;
|
|
5601
|
+
}
|
|
5602
|
+
}
|
|
5603
|
+
async function getParseFileFingerprint(adapter, filePath) {
|
|
5604
|
+
const primaryFingerprint = await createParseDependencyFingerprint(filePath, {
|
|
5605
|
+
allowMissing: false
|
|
5606
|
+
});
|
|
5607
|
+
if (!primaryFingerprint) {
|
|
5608
|
+
return void 0;
|
|
5609
|
+
}
|
|
5610
|
+
const additionalDependencyPaths = adapter.getParseDependencies ? await adapter.getParseDependencies(filePath) : [];
|
|
5611
|
+
const uniqueAdditionalDependencyPaths = [...new Set(additionalDependencyPaths)].filter((dependencyPath) => dependencyPath !== filePath).sort(compareByCodePoint);
|
|
5612
|
+
const dependencyFingerprints = [primaryFingerprint];
|
|
5613
|
+
for (const dependencyPath of uniqueAdditionalDependencyPaths) {
|
|
5614
|
+
const dependencyFingerprint = await createParseDependencyFingerprint(dependencyPath, {
|
|
5615
|
+
allowMissing: true
|
|
5616
|
+
});
|
|
5617
|
+
if (!dependencyFingerprint) {
|
|
5618
|
+
return void 0;
|
|
5619
|
+
}
|
|
5620
|
+
dependencyFingerprints.push(dependencyFingerprint);
|
|
5621
|
+
}
|
|
5622
|
+
return {
|
|
5623
|
+
dependencies: dependencyFingerprints
|
|
5624
|
+
};
|
|
5625
|
+
}
|
|
5626
|
+
function createParseBudget(maxParallelFileParsing) {
|
|
5627
|
+
const safeMaxParallelFileParsing = Number.isFinite(maxParallelFileParsing) && maxParallelFileParsing > 0 ? Math.max(1, Math.floor(maxParallelFileParsing)) : 1;
|
|
5628
|
+
let availablePermits = safeMaxParallelFileParsing;
|
|
5629
|
+
const waitingResolvers = [];
|
|
5630
|
+
async function acquire() {
|
|
5631
|
+
if (availablePermits > 0) {
|
|
5632
|
+
availablePermits -= 1;
|
|
5633
|
+
return;
|
|
5634
|
+
}
|
|
5635
|
+
await new Promise((resolve) => {
|
|
5636
|
+
waitingResolvers.push(resolve);
|
|
5637
|
+
});
|
|
5638
|
+
}
|
|
5639
|
+
function release() {
|
|
5640
|
+
const nextResolver = waitingResolvers.shift();
|
|
5641
|
+
if (nextResolver) {
|
|
5642
|
+
nextResolver();
|
|
5643
|
+
return;
|
|
5644
|
+
}
|
|
5645
|
+
availablePermits += 1;
|
|
5646
|
+
}
|
|
5647
|
+
return async (task) => {
|
|
5648
|
+
await acquire();
|
|
5649
|
+
try {
|
|
5650
|
+
return await task();
|
|
5651
|
+
} finally {
|
|
5652
|
+
release();
|
|
5653
|
+
}
|
|
5654
|
+
};
|
|
5655
|
+
}
|
|
5656
|
+
async function parseAdapterEvents(adapter, maxParallelFileParsing, runWithParseBudget = async (task) => task(), parseFileCache, runtimeProfile) {
|
|
5208
5657
|
const files = await adapter.discoverFiles();
|
|
5209
5658
|
if (files.length === 0) {
|
|
5210
5659
|
return {
|
|
@@ -5225,44 +5674,52 @@ async function parseAdapterEvents(adapter, maxParallelFileParsing, parseFileCach
|
|
|
5225
5674
|
while (nextFileIndex < files.length) {
|
|
5226
5675
|
const fileIndex = nextFileIndex;
|
|
5227
5676
|
nextFileIndex += 1;
|
|
5228
|
-
|
|
5229
|
-
|
|
5230
|
-
|
|
5231
|
-
|
|
5232
|
-
|
|
5233
|
-
|
|
5234
|
-
fileFingerprint
|
|
5235
|
-
|
|
5236
|
-
|
|
5237
|
-
|
|
5238
|
-
|
|
5239
|
-
|
|
5677
|
+
await runWithParseBudget(async () => {
|
|
5678
|
+
const filePath = files[fileIndex];
|
|
5679
|
+
let fileFingerprint;
|
|
5680
|
+
let parseFileDiagnostics;
|
|
5681
|
+
if (parseFileCache) {
|
|
5682
|
+
fileFingerprint = await getParseFileFingerprint(adapter, filePath);
|
|
5683
|
+
if (fileFingerprint) {
|
|
5684
|
+
parseFileDiagnostics = parseFileCache.get(adapter.id, filePath, fileFingerprint);
|
|
5685
|
+
runtimeProfile?.recordParseCacheResult(
|
|
5686
|
+
adapter.id,
|
|
5687
|
+
parseFileDiagnostics ? "hit" : "miss"
|
|
5688
|
+
);
|
|
5689
|
+
}
|
|
5240
5690
|
}
|
|
5241
|
-
|
|
5242
|
-
|
|
5243
|
-
|
|
5244
|
-
|
|
5245
|
-
|
|
5691
|
+
if (!parseFileDiagnostics) {
|
|
5692
|
+
parseFileDiagnostics = adapter.parseFileWithDiagnostics ? await adapter.parseFileWithDiagnostics(filePath) : getDefaultParseFileDiagnostics(await adapter.parseFile(filePath));
|
|
5693
|
+
if (parseFileCache && fileFingerprint) {
|
|
5694
|
+
parseFileCache.set(adapter.id, filePath, fileFingerprint, parseFileDiagnostics);
|
|
5695
|
+
}
|
|
5246
5696
|
}
|
|
5247
|
-
|
|
5248
|
-
|
|
5249
|
-
|
|
5250
|
-
|
|
5251
|
-
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5697
|
+
parsedByFile[fileIndex] = parseFileDiagnostics.events;
|
|
5698
|
+
skippedRowsByFile[fileIndex] = normalizeSkippedRowsCount(parseFileDiagnostics.skippedRows);
|
|
5699
|
+
for (const reasonStat of normalizeSkippedRowReasons(
|
|
5700
|
+
parseFileDiagnostics.skippedRowReasons
|
|
5701
|
+
)) {
|
|
5702
|
+
skippedRowReasons.set(
|
|
5703
|
+
reasonStat.reason,
|
|
5704
|
+
(skippedRowReasons.get(reasonStat.reason) ?? 0) + reasonStat.count
|
|
5705
|
+
);
|
|
5706
|
+
}
|
|
5707
|
+
});
|
|
5256
5708
|
}
|
|
5257
5709
|
});
|
|
5258
5710
|
await Promise.all(workers);
|
|
5259
|
-
|
|
5711
|
+
const result = {
|
|
5260
5712
|
source: adapter.id,
|
|
5261
5713
|
events: parsedByFile.flat(),
|
|
5262
5714
|
filesFound: files.length,
|
|
5263
5715
|
skippedRows: skippedRowsByFile.reduce((sum, skippedRowsCount) => sum + skippedRowsCount, 0),
|
|
5264
5716
|
skippedRowReasons: [...skippedRowReasons.entries()].map(([reason, count]) => ({ reason, count })).sort((left, right) => compareByCodePoint(left.reason, right.reason))
|
|
5265
5717
|
};
|
|
5718
|
+
runtimeProfile?.recordParseResult(adapter.id, {
|
|
5719
|
+
filesFound: result.filesFound,
|
|
5720
|
+
eventsParsed: result.events.length
|
|
5721
|
+
});
|
|
5722
|
+
return result;
|
|
5266
5723
|
}
|
|
5267
5724
|
function getErrorReason(error) {
|
|
5268
5725
|
if (error instanceof Error) {
|
|
@@ -5271,6 +5728,7 @@ function getErrorReason(error) {
|
|
|
5271
5728
|
return String(error);
|
|
5272
5729
|
}
|
|
5273
5730
|
async function parseSelectedAdapters(adaptersToParse, maxParallelFileParsing, options = {}) {
|
|
5731
|
+
const runWithParseBudget = createParseBudget(maxParallelFileParsing);
|
|
5274
5732
|
const parseCacheBySource = /* @__PURE__ */ new Map();
|
|
5275
5733
|
if (options.parseCache?.enabled) {
|
|
5276
5734
|
const parseCacheLimits = {
|
|
@@ -5279,12 +5737,9 @@ async function parseSelectedAdapters(adaptersToParse, maxParallelFileParsing, op
|
|
|
5279
5737
|
maxBytes: options.parseCache.maxBytes
|
|
5280
5738
|
};
|
|
5281
5739
|
const cacheFilePath = options.parseCacheFilePath ?? getDefaultParseFileCachePath();
|
|
5740
|
+
const sourceIds = [...new Set(adaptersToParse.map((adapter) => adapter.id.toLowerCase()))];
|
|
5282
5741
|
await Promise.all(
|
|
5283
|
-
|
|
5284
|
-
const sourceId = adapter.id.toLowerCase();
|
|
5285
|
-
if (parseCacheBySource.has(sourceId)) {
|
|
5286
|
-
return;
|
|
5287
|
-
}
|
|
5742
|
+
sourceIds.map(async (sourceId) => {
|
|
5288
5743
|
parseCacheBySource.set(
|
|
5289
5744
|
sourceId,
|
|
5290
5745
|
await ParseFileCache.load({
|
|
@@ -5298,9 +5753,19 @@ async function parseSelectedAdapters(adaptersToParse, maxParallelFileParsing, op
|
|
|
5298
5753
|
}
|
|
5299
5754
|
const parseResults = await Promise.allSettled(
|
|
5300
5755
|
adaptersToParse.map(
|
|
5301
|
-
(adapter) =>
|
|
5756
|
+
(adapter) => options.runtimeProfile ? options.runtimeProfile.measure(
|
|
5757
|
+
`parse.adapter.${adapter.id}`,
|
|
5758
|
+
() => parseAdapterEvents(
|
|
5759
|
+
adapter,
|
|
5760
|
+
maxParallelFileParsing,
|
|
5761
|
+
runWithParseBudget,
|
|
5762
|
+
parseCacheBySource.get(adapter.id.toLowerCase()),
|
|
5763
|
+
options.runtimeProfile
|
|
5764
|
+
)
|
|
5765
|
+
) : parseAdapterEvents(
|
|
5302
5766
|
adapter,
|
|
5303
5767
|
maxParallelFileParsing,
|
|
5768
|
+
runWithParseBudget,
|
|
5304
5769
|
parseCacheBySource.get(adapter.id.toLowerCase())
|
|
5305
5770
|
)
|
|
5306
5771
|
)
|
|
@@ -5335,12 +5800,6 @@ function throwOnExplicitSourceFailures(sourceFailures, explicitSourceIds) {
|
|
|
5335
5800
|
const details = explicitFailures.map((failure) => `${failure.source}: ${failure.reason}`).join("; ");
|
|
5336
5801
|
throw new Error(`Failed to parse explicitly requested source(s): ${details}`);
|
|
5337
5802
|
}
|
|
5338
|
-
function matchesProvider(provider, providerFilter) {
|
|
5339
|
-
if (!providerFilter) {
|
|
5340
|
-
return true;
|
|
5341
|
-
}
|
|
5342
|
-
return provider?.toLowerCase().includes(providerFilter) ?? false;
|
|
5343
|
-
}
|
|
5344
5803
|
function isEventWithinDateRange(event, timezone, since, until) {
|
|
5345
5804
|
const eventDate = getPeriodKey(event.timestamp, "daily", timezone);
|
|
5346
5805
|
if (since && eventDate < since) {
|
|
@@ -5379,19 +5838,26 @@ function filterByModelRules(events, modelFilter) {
|
|
|
5379
5838
|
const modelFilterRules = resolveModelFilterRules(events, modelFilter);
|
|
5380
5839
|
return events.filter((event) => matchesModel(event.model, modelFilterRules));
|
|
5381
5840
|
}
|
|
5382
|
-
function
|
|
5383
|
-
const
|
|
5384
|
-
for (const
|
|
5385
|
-
for (const event of
|
|
5386
|
-
if (!
|
|
5841
|
+
function collectProviderAndDateFilteredEvents(eventGroups, options) {
|
|
5842
|
+
const filteredEvents = [];
|
|
5843
|
+
for (const events of eventGroups) {
|
|
5844
|
+
for (const event of events) {
|
|
5845
|
+
if (!matchesCanonicalProviderFilter(event.provider, options.providerFilter)) {
|
|
5387
5846
|
continue;
|
|
5388
5847
|
}
|
|
5389
5848
|
if (!isEventWithinDateRange(event, options.timezone, options.since, options.until)) {
|
|
5390
5849
|
continue;
|
|
5391
5850
|
}
|
|
5392
|
-
|
|
5851
|
+
filteredEvents.push(event);
|
|
5393
5852
|
}
|
|
5394
5853
|
}
|
|
5854
|
+
return filteredEvents;
|
|
5855
|
+
}
|
|
5856
|
+
function filterParsedAdapterEvents(parseResults, options) {
|
|
5857
|
+
const providerAndDateFilteredEvents = collectProviderAndDateFilteredEvents(
|
|
5858
|
+
parseResults.map((result) => result.events),
|
|
5859
|
+
options
|
|
5860
|
+
);
|
|
5395
5861
|
return filterByModelRules(providerAndDateFilteredEvents, options.modelFilter);
|
|
5396
5862
|
}
|
|
5397
5863
|
|
|
@@ -5412,21 +5878,63 @@ function calculateEstimatedCostUsd(event, pricing) {
|
|
|
5412
5878
|
const reasoningCost = reasoningBilling === "separate" ? estimateTokenGroupCost(event.reasoningTokens, pricing.reasoningPer1MUsd) : 0;
|
|
5413
5879
|
return inputCost + outputCost + cacheReadCost + cacheWriteCost + reasoningCost;
|
|
5414
5880
|
}
|
|
5415
|
-
function
|
|
5416
|
-
|
|
5881
|
+
function hasNonReasoningPricedBuckets(usage) {
|
|
5882
|
+
return usage.inputTokens > 0 || usage.outputTokens > 0 || usage.cacheReadTokens > 0 || usage.cacheWriteTokens > 0;
|
|
5883
|
+
}
|
|
5884
|
+
function hasDefinedRate(rate) {
|
|
5885
|
+
return rate !== void 0;
|
|
5886
|
+
}
|
|
5887
|
+
function hasUsedBucketWithUndefinedRate(usage, pricing) {
|
|
5888
|
+
if (usage.inputTokens > 0 && !hasDefinedRate(pricing.inputPer1MUsd)) {
|
|
5889
|
+
return true;
|
|
5890
|
+
}
|
|
5891
|
+
if (usage.outputTokens > 0 && !hasDefinedRate(pricing.outputPer1MUsd)) {
|
|
5892
|
+
return true;
|
|
5893
|
+
}
|
|
5894
|
+
if (usage.cacheReadTokens > 0 && !hasDefinedRate(pricing.cacheReadPer1MUsd)) {
|
|
5895
|
+
return true;
|
|
5896
|
+
}
|
|
5897
|
+
if (usage.cacheWriteTokens > 0 && !hasDefinedRate(pricing.cacheWritePer1MUsd)) {
|
|
5898
|
+
return true;
|
|
5899
|
+
}
|
|
5900
|
+
const reasoningBilling = pricing.reasoningBilling ?? "included-in-output";
|
|
5901
|
+
return usage.reasoningTokens > 0 && reasoningBilling === "separate" && !hasDefinedRate(pricing.reasoningPer1MUsd);
|
|
5902
|
+
}
|
|
5903
|
+
function canEstimateUsageCost(usage, pricing) {
|
|
5904
|
+
if (hasUsedBucketWithUndefinedRate(usage, pricing)) {
|
|
5905
|
+
return false;
|
|
5906
|
+
}
|
|
5907
|
+
const reasoningBilling = pricing.reasoningBilling ?? "included-in-output";
|
|
5908
|
+
if (usage.reasoningTokens > 0) {
|
|
5909
|
+
if (reasoningBilling === "separate") {
|
|
5910
|
+
return true;
|
|
5911
|
+
}
|
|
5912
|
+
if (usage.outputTokens === 0) {
|
|
5913
|
+
return false;
|
|
5914
|
+
}
|
|
5915
|
+
}
|
|
5916
|
+
return hasNonReasoningPricedBuckets(usage);
|
|
5917
|
+
}
|
|
5918
|
+
function applyResolvedPricingToEvent(event, pricing) {
|
|
5919
|
+
const shouldRepriceExplicitZero = event.costMode === "explicit" && event.costUsd === 0 && event.model !== void 0 && isPriceableEvent(event);
|
|
5417
5920
|
if (event.costMode === "explicit" && event.costUsd !== void 0 && !shouldRepriceExplicitZero) {
|
|
5418
5921
|
return event;
|
|
5419
5922
|
}
|
|
5420
|
-
if (!event.model) {
|
|
5923
|
+
if (!event.model || !isPriceableEvent(event)) {
|
|
5421
5924
|
return { ...event, costMode: "estimated" };
|
|
5422
5925
|
}
|
|
5423
|
-
const pricing = pricingSource.getPricing(event.model);
|
|
5424
5926
|
if (!pricing) {
|
|
5425
5927
|
if (shouldRepriceExplicitZero) {
|
|
5426
5928
|
return event;
|
|
5427
5929
|
}
|
|
5428
5930
|
return { ...event, costMode: "estimated" };
|
|
5429
5931
|
}
|
|
5932
|
+
if (!canEstimateUsageCost(event, pricing)) {
|
|
5933
|
+
if (shouldRepriceExplicitZero) {
|
|
5934
|
+
return event;
|
|
5935
|
+
}
|
|
5936
|
+
return { ...event, costMode: "estimated" };
|
|
5937
|
+
}
|
|
5430
5938
|
return {
|
|
5431
5939
|
...event,
|
|
5432
5940
|
costUsd: calculateEstimatedCostUsd(event, pricing),
|
|
@@ -5434,7 +5942,20 @@ function applyPricingToEvent(event, pricingSource) {
|
|
|
5434
5942
|
};
|
|
5435
5943
|
}
|
|
5436
5944
|
function applyPricingToEvents(events, pricingSource) {
|
|
5437
|
-
|
|
5945
|
+
const pricingByModel = /* @__PURE__ */ new Map();
|
|
5946
|
+
return events.map((event) => {
|
|
5947
|
+
const model = event.model;
|
|
5948
|
+
let pricing;
|
|
5949
|
+
if (model && isPriceableEvent(event)) {
|
|
5950
|
+
const resolvedModel = pricingSource.resolveModelAlias(model);
|
|
5951
|
+
pricing = pricingByModel.get(resolvedModel);
|
|
5952
|
+
if (!pricingByModel.has(resolvedModel)) {
|
|
5953
|
+
pricing = pricingSource.getPricing(resolvedModel);
|
|
5954
|
+
pricingByModel.set(resolvedModel, pricing);
|
|
5955
|
+
}
|
|
5956
|
+
}
|
|
5957
|
+
return applyResolvedPricingToEvent(event, pricing);
|
|
5958
|
+
});
|
|
5438
5959
|
}
|
|
5439
5960
|
|
|
5440
5961
|
// src/pricing/litellm-pricing-fetcher.ts
|
|
@@ -5481,8 +6002,8 @@ var litellm_model_map_default = {
|
|
|
5481
6002
|
|
|
5482
6003
|
// src/pricing/litellm-pricing-fetcher.ts
|
|
5483
6004
|
var ONE_MILLION2 = 1e6;
|
|
5484
|
-
var
|
|
5485
|
-
var
|
|
6005
|
+
var DEFAULT_CACHE_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
6006
|
+
var DEFAULT_FETCH_TIMEOUT_MS = 4e3;
|
|
5486
6007
|
var DEFAULT_FETCH_RETRY_COUNT2 = 2;
|
|
5487
6008
|
var DEFAULT_FETCH_RETRY_DELAY_MS2 = 200;
|
|
5488
6009
|
var DEFAULT_LITELLM_PRICING_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";
|
|
@@ -5707,8 +6228,8 @@ var LiteLLMPricingFetcher = class {
|
|
|
5707
6228
|
constructor(options = {}) {
|
|
5708
6229
|
this.sourceUrl = options.sourceUrl ?? DEFAULT_LITELLM_PRICING_URL;
|
|
5709
6230
|
this.cacheFilePath = options.cacheFilePath ?? getDefaultLiteLLMPricingCachePath();
|
|
5710
|
-
this.cacheTtlMs = options.cacheTtlMs ??
|
|
5711
|
-
this.fetchTimeoutMs = options.fetchTimeoutMs ??
|
|
6231
|
+
this.cacheTtlMs = options.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS;
|
|
6232
|
+
this.fetchTimeoutMs = options.fetchTimeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
|
|
5712
6233
|
this.fetchRetryCount = Number.isFinite(options.fetchRetryCount) && (options.fetchRetryCount ?? 0) >= 0 ? Math.trunc(options.fetchRetryCount ?? DEFAULT_FETCH_RETRY_COUNT2) : DEFAULT_FETCH_RETRY_COUNT2;
|
|
5713
6234
|
this.fetchRetryDelayMs = Number.isFinite(options.fetchRetryDelayMs) && (options.fetchRetryDelayMs ?? 0) > 0 ? Math.trunc(options.fetchRetryDelayMs ?? DEFAULT_FETCH_RETRY_DELAY_MS2) : DEFAULT_FETCH_RETRY_DELAY_MS2;
|
|
5714
6235
|
this.offline = options.offline ?? false;
|
|
@@ -6061,7 +6582,7 @@ function eventNeedsPricingLookup(event) {
|
|
|
6061
6582
|
if (!event.model) {
|
|
6062
6583
|
return false;
|
|
6063
6584
|
}
|
|
6064
|
-
if (event
|
|
6585
|
+
if (!isPriceableEvent(event)) {
|
|
6065
6586
|
return false;
|
|
6066
6587
|
}
|
|
6067
6588
|
return event.costMode !== "explicit" || event.costUsd === void 0 || event.costUsd === 0;
|
|
@@ -6072,14 +6593,9 @@ function shouldLoadPricingSource(events) {
|
|
|
6072
6593
|
}
|
|
6073
6594
|
return events.some((event) => eventNeedsPricingLookup(event));
|
|
6074
6595
|
}
|
|
6075
|
-
function hasAnyBillableTokenBuckets(events) {
|
|
6076
|
-
return events.some(
|
|
6077
|
-
(event) => event.inputTokens > 0 || event.outputTokens > 0 || event.reasoningTokens > 0 || event.cacheReadTokens > 0 || event.cacheWriteTokens > 0
|
|
6078
|
-
);
|
|
6079
|
-
}
|
|
6080
6596
|
function shouldLoadPricingSourceForMode(events, pricingLoadMode) {
|
|
6081
6597
|
if (pricingLoadMode === "force") {
|
|
6082
|
-
return
|
|
6598
|
+
return events.some((event) => isPriceableEvent(event));
|
|
6083
6599
|
}
|
|
6084
6600
|
return shouldLoadPricingSource(events);
|
|
6085
6601
|
}
|
|
@@ -6114,6 +6630,195 @@ async function resolveAndApplyPricingToEvents(events, options, runtimeConfig = g
|
|
|
6114
6630
|
};
|
|
6115
6631
|
}
|
|
6116
6632
|
|
|
6633
|
+
// src/cli/runtime-profile.ts
|
|
6634
|
+
var RUNTIME_PROFILE_ENV_VAR = "LLM_USAGE_PROFILE_RUNTIME";
|
|
6635
|
+
function isTruthyEnvFlag2(value) {
|
|
6636
|
+
if (value === void 0) {
|
|
6637
|
+
return false;
|
|
6638
|
+
}
|
|
6639
|
+
const normalizedValue = value.trim().toLowerCase();
|
|
6640
|
+
if (normalizedValue.length === 0) {
|
|
6641
|
+
return false;
|
|
6642
|
+
}
|
|
6643
|
+
return ["1", "true", "yes", "on"].includes(normalizedValue);
|
|
6644
|
+
}
|
|
6645
|
+
function isRuntimeProfileEnabled(env = process.env) {
|
|
6646
|
+
return isTruthyEnvFlag2(env[RUNTIME_PROFILE_ENV_VAR]);
|
|
6647
|
+
}
|
|
6648
|
+
var RuntimeProfileCollector = class {
|
|
6649
|
+
constructor(now = () => performance.now()) {
|
|
6650
|
+
this.now = now;
|
|
6651
|
+
}
|
|
6652
|
+
sourceSelection;
|
|
6653
|
+
sourceStats = /* @__PURE__ */ new Map();
|
|
6654
|
+
stageDurations = /* @__PURE__ */ new Map();
|
|
6655
|
+
async measure(name, task) {
|
|
6656
|
+
const startedAt = this.now();
|
|
6657
|
+
try {
|
|
6658
|
+
return await task();
|
|
6659
|
+
} finally {
|
|
6660
|
+
this.recordStageDuration(name, this.now() - startedAt);
|
|
6661
|
+
}
|
|
6662
|
+
}
|
|
6663
|
+
measureSync(name, task) {
|
|
6664
|
+
const startedAt = this.now();
|
|
6665
|
+
try {
|
|
6666
|
+
return task();
|
|
6667
|
+
} finally {
|
|
6668
|
+
this.recordStageDuration(name, this.now() - startedAt);
|
|
6669
|
+
}
|
|
6670
|
+
}
|
|
6671
|
+
recordStageDuration(name, durationMs) {
|
|
6672
|
+
if (!name || !Number.isFinite(durationMs) || durationMs < 0) {
|
|
6673
|
+
return;
|
|
6674
|
+
}
|
|
6675
|
+
this.stageDurations.set(name, (this.stageDurations.get(name) ?? 0) + durationMs);
|
|
6676
|
+
}
|
|
6677
|
+
recordSourceSelection(selection) {
|
|
6678
|
+
this.sourceSelection = {
|
|
6679
|
+
availableSourceIds: [...selection.availableSourceIds],
|
|
6680
|
+
selectedSourceIds: [...selection.selectedSourceIds],
|
|
6681
|
+
candidateProviderRoots: selection.candidateProviderRoots ? [...selection.candidateProviderRoots] : void 0
|
|
6682
|
+
};
|
|
6683
|
+
}
|
|
6684
|
+
recordParseCacheResult(source, result) {
|
|
6685
|
+
const sourceStats = this.getOrCreateSourceStats(source);
|
|
6686
|
+
if (result === "hit") {
|
|
6687
|
+
sourceStats.cacheHits += 1;
|
|
6688
|
+
return;
|
|
6689
|
+
}
|
|
6690
|
+
sourceStats.cacheMisses += 1;
|
|
6691
|
+
}
|
|
6692
|
+
recordParseResult(source, result) {
|
|
6693
|
+
const sourceStats = this.getOrCreateSourceStats(source);
|
|
6694
|
+
sourceStats.filesFound += Math.max(0, Math.trunc(result.filesFound));
|
|
6695
|
+
sourceStats.eventsParsed += Math.max(0, Math.trunc(result.eventsParsed));
|
|
6696
|
+
}
|
|
6697
|
+
snapshot() {
|
|
6698
|
+
const sourceStats = [...this.sourceStats.values()].sort(
|
|
6699
|
+
(left, right) => compareByCodePoint(left.source, right.source)
|
|
6700
|
+
);
|
|
6701
|
+
const parseTotals = sourceStats.reduce(
|
|
6702
|
+
(totals, source) => ({
|
|
6703
|
+
filesFound: totals.filesFound + source.filesFound,
|
|
6704
|
+
eventsParsed: totals.eventsParsed + source.eventsParsed
|
|
6705
|
+
}),
|
|
6706
|
+
{ filesFound: 0, eventsParsed: 0 }
|
|
6707
|
+
);
|
|
6708
|
+
const parseCache = sourceStats.reduce(
|
|
6709
|
+
(totals, source) => ({
|
|
6710
|
+
hits: totals.hits + source.cacheHits,
|
|
6711
|
+
misses: totals.misses + source.cacheMisses
|
|
6712
|
+
}),
|
|
6713
|
+
{ hits: 0, misses: 0 }
|
|
6714
|
+
);
|
|
6715
|
+
const stageTimings = [...this.stageDurations.entries()].sort(([leftName], [rightName]) => compareByCodePoint(leftName, rightName)).map(([name, durationMs]) => ({ name, durationMs }));
|
|
6716
|
+
return {
|
|
6717
|
+
sourceSelection: this.sourceSelection ? {
|
|
6718
|
+
availableSourceIds: [...this.sourceSelection.availableSourceIds],
|
|
6719
|
+
selectedSourceIds: [...this.sourceSelection.selectedSourceIds],
|
|
6720
|
+
candidateProviderRoots: this.sourceSelection.candidateProviderRoots ? [...this.sourceSelection.candidateProviderRoots] : void 0
|
|
6721
|
+
} : void 0,
|
|
6722
|
+
parseCache,
|
|
6723
|
+
parseTotals,
|
|
6724
|
+
sourceStats: sourceStats.map((source) => ({ ...source })),
|
|
6725
|
+
stageTimings
|
|
6726
|
+
};
|
|
6727
|
+
}
|
|
6728
|
+
getOrCreateSourceStats(source) {
|
|
6729
|
+
const existing = this.sourceStats.get(source);
|
|
6730
|
+
if (existing) {
|
|
6731
|
+
return existing;
|
|
6732
|
+
}
|
|
6733
|
+
const created = {
|
|
6734
|
+
source,
|
|
6735
|
+
filesFound: 0,
|
|
6736
|
+
eventsParsed: 0,
|
|
6737
|
+
cacheHits: 0,
|
|
6738
|
+
cacheMisses: 0
|
|
6739
|
+
};
|
|
6740
|
+
this.sourceStats.set(source, created);
|
|
6741
|
+
return created;
|
|
6742
|
+
}
|
|
6743
|
+
};
|
|
6744
|
+
async function measureRuntimeProfileStage(runtimeProfile, name, task) {
|
|
6745
|
+
if (!runtimeProfile) {
|
|
6746
|
+
return task();
|
|
6747
|
+
}
|
|
6748
|
+
return runtimeProfile.measure(name, task);
|
|
6749
|
+
}
|
|
6750
|
+
function measureRuntimeProfileStageSync(runtimeProfile, name, task) {
|
|
6751
|
+
if (!runtimeProfile) {
|
|
6752
|
+
return task();
|
|
6753
|
+
}
|
|
6754
|
+
return runtimeProfile.measureSync(name, task);
|
|
6755
|
+
}
|
|
6756
|
+
function createRuntimeProfileCollector(env = process.env) {
|
|
6757
|
+
if (!isRuntimeProfileEnabled(env)) {
|
|
6758
|
+
return void 0;
|
|
6759
|
+
}
|
|
6760
|
+
return new RuntimeProfileCollector();
|
|
6761
|
+
}
|
|
6762
|
+
function hasRecordedSourceStats(snapshot) {
|
|
6763
|
+
return snapshot.sourceStats.length > 0;
|
|
6764
|
+
}
|
|
6765
|
+
function hasRecordedParseCache(snapshot) {
|
|
6766
|
+
return snapshot.parseCache.hits > 0 || snapshot.parseCache.misses > 0 || hasRecordedSourceStats(snapshot);
|
|
6767
|
+
}
|
|
6768
|
+
function hasRecordedParseTotals(snapshot) {
|
|
6769
|
+
return snapshot.parseTotals.filesFound > 0 || snapshot.parseTotals.eventsParsed > 0 || hasRecordedSourceStats(snapshot);
|
|
6770
|
+
}
|
|
6771
|
+
function mergeRuntimeProfiles(primary, fallback) {
|
|
6772
|
+
if (!primary) {
|
|
6773
|
+
return fallback;
|
|
6774
|
+
}
|
|
6775
|
+
if (!fallback) {
|
|
6776
|
+
return primary;
|
|
6777
|
+
}
|
|
6778
|
+
const stageTimingsByName = new Map(
|
|
6779
|
+
fallback.stageTimings.map((stageTiming) => [stageTiming.name, stageTiming])
|
|
6780
|
+
);
|
|
6781
|
+
for (const stageTiming of primary.stageTimings) {
|
|
6782
|
+
stageTimingsByName.set(stageTiming.name, stageTiming);
|
|
6783
|
+
}
|
|
6784
|
+
return {
|
|
6785
|
+
sourceSelection: primary.sourceSelection ?? fallback.sourceSelection,
|
|
6786
|
+
parseCache: hasRecordedParseCache(primary) ? primary.parseCache : fallback.parseCache,
|
|
6787
|
+
parseTotals: hasRecordedParseTotals(primary) ? primary.parseTotals : fallback.parseTotals,
|
|
6788
|
+
sourceStats: primary.sourceStats.length > 0 ? primary.sourceStats : fallback.sourceStats,
|
|
6789
|
+
stageTimings: [...stageTimingsByName.values()].sort(
|
|
6790
|
+
(left, right) => compareByCodePoint(left.name, right.name)
|
|
6791
|
+
)
|
|
6792
|
+
};
|
|
6793
|
+
}
|
|
6794
|
+
function emitRuntimeProfile(runtimeProfile, diagnosticsLogger = logger) {
|
|
6795
|
+
if (!runtimeProfile) {
|
|
6796
|
+
return;
|
|
6797
|
+
}
|
|
6798
|
+
diagnosticsLogger.info("Runtime profile:");
|
|
6799
|
+
if (runtimeProfile.sourceSelection) {
|
|
6800
|
+
const candidateProviderRoots = runtimeProfile.sourceSelection.candidateProviderRoots && runtimeProfile.sourceSelection.candidateProviderRoots.length > 0 ? `; candidateProviderRoots=${runtimeProfile.sourceSelection.candidateProviderRoots.join(",")}` : "";
|
|
6801
|
+
diagnosticsLogger.dim(
|
|
6802
|
+
` source selection: available=${runtimeProfile.sourceSelection.availableSourceIds.join(",")}; selected=${runtimeProfile.sourceSelection.selectedSourceIds.join(",")}${candidateProviderRoots}`
|
|
6803
|
+
);
|
|
6804
|
+
}
|
|
6805
|
+
diagnosticsLogger.dim(
|
|
6806
|
+
` parse cache: hits=${runtimeProfile.parseCache.hits}; misses=${runtimeProfile.parseCache.misses}`
|
|
6807
|
+
);
|
|
6808
|
+
diagnosticsLogger.dim(
|
|
6809
|
+
` parse totals: files=${runtimeProfile.parseTotals.filesFound}; events=${runtimeProfile.parseTotals.eventsParsed}`
|
|
6810
|
+
);
|
|
6811
|
+
for (const source of runtimeProfile.sourceStats) {
|
|
6812
|
+
diagnosticsLogger.dim(
|
|
6813
|
+
` source ${source.source}: files=${source.filesFound}; events=${source.eventsParsed}; cacheHits=${source.cacheHits}; cacheMisses=${source.cacheMisses}`
|
|
6814
|
+
);
|
|
6815
|
+
}
|
|
6816
|
+
diagnosticsLogger.dim(" stage timings:");
|
|
6817
|
+
for (const stageTiming of runtimeProfile.stageTimings) {
|
|
6818
|
+
diagnosticsLogger.dim(` ${stageTiming.name}: ${stageTiming.durationMs.toFixed(2)}ms`);
|
|
6819
|
+
}
|
|
6820
|
+
}
|
|
6821
|
+
|
|
6117
6822
|
// src/cli/build-usage-event-dataset.ts
|
|
6118
6823
|
function withNormalizedPricingUrl(options, normalizedPricingUrl) {
|
|
6119
6824
|
if (options.pricingUrl === normalizedPricingUrl) {
|
|
@@ -6126,33 +6831,57 @@ function withNormalizedPricingUrl(options, normalizedPricingUrl) {
|
|
|
6126
6831
|
}
|
|
6127
6832
|
async function buildUsageEventDataset(options, deps = {}) {
|
|
6128
6833
|
const normalizedInputs = normalizeBuildUsageInputs(options);
|
|
6834
|
+
const runtimeProfile = deps.runtimeProfile;
|
|
6129
6835
|
const readParsingRuntimeConfig = deps.getParsingRuntimeConfig ?? getParsingRuntimeConfig;
|
|
6130
6836
|
const readPricingRuntimeConfig = deps.getPricingFetcherRuntimeConfig ?? getPricingFetcherRuntimeConfig;
|
|
6131
6837
|
const makeAdapters = deps.createAdapters ?? createDefaultAdapters;
|
|
6132
6838
|
const parsingRuntimeConfig = readParsingRuntimeConfig();
|
|
6133
6839
|
const pricingRuntimeConfig = readPricingRuntimeConfig();
|
|
6134
|
-
const adapters =
|
|
6135
|
-
|
|
6136
|
-
|
|
6137
|
-
|
|
6138
|
-
|
|
6139
|
-
|
|
6840
|
+
const adapters = measureRuntimeProfileStageSync(
|
|
6841
|
+
runtimeProfile,
|
|
6842
|
+
"usage.dataset.create_adapters",
|
|
6843
|
+
() => makeAdapters(options)
|
|
6844
|
+
);
|
|
6845
|
+
const adaptersToParse = measureRuntimeProfileStageSync(
|
|
6846
|
+
runtimeProfile,
|
|
6847
|
+
"usage.dataset.select_adapters",
|
|
6848
|
+
() => selectAdaptersForParsing(adapters, {
|
|
6849
|
+
sourceFilter: normalizedInputs.sourceFilter,
|
|
6850
|
+
candidateProviderRoots: normalizedInputs.candidateProviderRoots,
|
|
6851
|
+
runtimeProfile
|
|
6852
|
+
})
|
|
6853
|
+
);
|
|
6854
|
+
throwOnExplicitSourceScopeConflicts(adapters, adaptersToParse, {
|
|
6855
|
+
explicitSourceIds: normalizedInputs.explicitSourceIds,
|
|
6856
|
+
candidateProviderRoots: normalizedInputs.candidateProviderRoots,
|
|
6857
|
+
providerFilter: normalizedInputs.providerFilter,
|
|
6858
|
+
modelFilter: normalizedInputs.modelFilter
|
|
6859
|
+
});
|
|
6860
|
+
const { successfulParseResults, sourceFailures } = await measureRuntimeProfileStage(
|
|
6861
|
+
runtimeProfile,
|
|
6862
|
+
"usage.dataset.parse_adapters",
|
|
6863
|
+
() => parseSelectedAdapters(adaptersToParse, parsingRuntimeConfig.maxParallelFileParsing, {
|
|
6140
6864
|
parseCache: {
|
|
6141
6865
|
enabled: parsingRuntimeConfig.parseCacheEnabled,
|
|
6142
6866
|
ttlMs: parsingRuntimeConfig.parseCacheTtlMs,
|
|
6143
6867
|
maxEntries: parsingRuntimeConfig.parseCacheMaxEntries,
|
|
6144
6868
|
maxBytes: parsingRuntimeConfig.parseCacheMaxBytes
|
|
6145
|
-
}
|
|
6146
|
-
|
|
6869
|
+
},
|
|
6870
|
+
runtimeProfile
|
|
6871
|
+
})
|
|
6147
6872
|
);
|
|
6148
6873
|
throwOnExplicitSourceFailures(sourceFailures, normalizedInputs.explicitSourceIds);
|
|
6149
|
-
const filteredEvents =
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
|
|
6155
|
-
|
|
6874
|
+
const filteredEvents = measureRuntimeProfileStageSync(
|
|
6875
|
+
runtimeProfile,
|
|
6876
|
+
"usage.dataset.filter_events",
|
|
6877
|
+
() => filterParsedAdapterEvents(successfulParseResults, {
|
|
6878
|
+
timezone: normalizedInputs.timezone,
|
|
6879
|
+
since: options.since,
|
|
6880
|
+
until: options.until,
|
|
6881
|
+
providerFilter: normalizedInputs.providerFilter,
|
|
6882
|
+
modelFilter: normalizedInputs.modelFilter
|
|
6883
|
+
})
|
|
6884
|
+
);
|
|
6156
6885
|
return {
|
|
6157
6886
|
options,
|
|
6158
6887
|
normalizedInputs,
|
|
@@ -6170,38 +6899,17 @@ async function applyPricingToUsageEventDataset(dataset, deps = {}, pricingLoadMo
|
|
|
6170
6899
|
dataset.options,
|
|
6171
6900
|
dataset.normalizedInputs.pricingUrl
|
|
6172
6901
|
);
|
|
6173
|
-
return
|
|
6174
|
-
|
|
6175
|
-
|
|
6176
|
-
|
|
6177
|
-
|
|
6178
|
-
|
|
6179
|
-
|
|
6180
|
-
|
|
6181
|
-
|
|
6182
|
-
|
|
6183
|
-
async function buildUsageData(granularity, options, deps = {}) {
|
|
6184
|
-
const dataset = await buildUsageEventDataset(options, deps);
|
|
6185
|
-
const { pricedEvents, pricingOrigin, pricingWarning } = await applyPricingToUsageEventDataset(
|
|
6186
|
-
dataset,
|
|
6187
|
-
deps,
|
|
6188
|
-
"auto"
|
|
6902
|
+
return measureRuntimeProfileStage(
|
|
6903
|
+
deps.runtimeProfile,
|
|
6904
|
+
"usage.pricing.apply",
|
|
6905
|
+
() => resolveAndApplyPricingToEvents(
|
|
6906
|
+
dataset.filteredEvents,
|
|
6907
|
+
pricingOptions,
|
|
6908
|
+
dataset.pricingRuntimeConfig,
|
|
6909
|
+
loadPricingSource,
|
|
6910
|
+
pricingLoadMode
|
|
6911
|
+
)
|
|
6189
6912
|
);
|
|
6190
|
-
const rows = aggregateUsage(pricedEvents, {
|
|
6191
|
-
granularity,
|
|
6192
|
-
timezone: dataset.normalizedInputs.timezone,
|
|
6193
|
-
sourceOrder: dataset.adaptersToParse.map((adapter) => adapter.id)
|
|
6194
|
-
});
|
|
6195
|
-
const diagnostics = buildUsageDiagnostics({
|
|
6196
|
-
adaptersToParse: dataset.adaptersToParse,
|
|
6197
|
-
successfulParseResults: dataset.successfulParseResults,
|
|
6198
|
-
sourceFailures: dataset.sourceFailures,
|
|
6199
|
-
pricingOrigin,
|
|
6200
|
-
pricingWarning,
|
|
6201
|
-
activeEnvOverrides: dataset.readEnvVarOverrides(),
|
|
6202
|
-
timezone: dataset.normalizedInputs.timezone
|
|
6203
|
-
});
|
|
6204
|
-
return assembleUsageDataResult(pricedEvents, rows, diagnostics);
|
|
6205
6913
|
}
|
|
6206
6914
|
|
|
6207
6915
|
// src/cli/build-efficiency-data.ts
|
|
@@ -6255,52 +6963,80 @@ function resolveScopeNote(options) {
|
|
|
6255
6963
|
return `Usage filters (${activeFilters.join(", ")}) affect commit attribution too: only commit days with matching repo-attributed usage events are counted.`;
|
|
6256
6964
|
}
|
|
6257
6965
|
function hasMeaningfulEfficiencyUsageSignal(event) {
|
|
6258
|
-
return event.totalTokens > 0 || event.costUsd !== void 0
|
|
6966
|
+
return event.totalTokens > 0 || hasBillableTokenBuckets(event) || event.costUsd !== void 0;
|
|
6259
6967
|
}
|
|
6260
6968
|
async function buildEfficiencyData(granularity, options, deps = {}) {
|
|
6261
|
-
const
|
|
6969
|
+
const buildDataset = deps.buildUsageEventDataset ?? buildUsageEventDataset;
|
|
6970
|
+
const applyPricing = deps.applyPricingToUsageEventDataset ?? applyPricingToUsageEventDataset;
|
|
6262
6971
|
const collectOutcomes = deps.collectGitOutcomes ?? collectGitOutcomes;
|
|
6263
6972
|
const resolveRepoRoot3 = deps.resolveRepoRoot ?? resolveRepoRootFromPathHint;
|
|
6264
6973
|
const repoDir = options.repoDir?.trim();
|
|
6265
6974
|
if (options.repoDir !== void 0 && !repoDir) {
|
|
6266
6975
|
throw new Error("--repo-dir must be a non-empty path");
|
|
6267
6976
|
}
|
|
6268
|
-
const
|
|
6269
|
-
|
|
6270
|
-
|
|
6271
|
-
|
|
6272
|
-
|
|
6977
|
+
const dataset = await measureRuntimeProfileStage(
|
|
6978
|
+
deps.runtimeProfile,
|
|
6979
|
+
"efficiency.dataset.total",
|
|
6980
|
+
() => buildDataset(options, deps)
|
|
6981
|
+
);
|
|
6982
|
+
const { pricedEvents, pricingOrigin, pricingWarning } = await applyPricing(dataset, deps, "auto");
|
|
6983
|
+
const attribution = await measureRuntimeProfileStage(
|
|
6984
|
+
deps.runtimeProfile,
|
|
6985
|
+
"efficiency.attribute_repo",
|
|
6986
|
+
() => attributeUsageEventsToRepo(pricedEvents, repoDir ?? process.cwd(), resolveRepoRoot3)
|
|
6273
6987
|
);
|
|
6274
6988
|
const matchedEventsWithSignal = attribution.matchedEvents.filter(
|
|
6275
6989
|
(event) => hasMeaningfulEfficiencyUsageSignal(event)
|
|
6276
6990
|
);
|
|
6277
6991
|
const activeUsageDays = new Set(
|
|
6278
6992
|
matchedEventsWithSignal.map(
|
|
6279
|
-
(event) => getPeriodKey(event.timestamp, "daily",
|
|
6993
|
+
(event) => getPeriodKey(event.timestamp, "daily", dataset.normalizedInputs.timezone)
|
|
6280
6994
|
)
|
|
6281
6995
|
);
|
|
6282
|
-
const gitOutcomes = await
|
|
6283
|
-
|
|
6284
|
-
|
|
6285
|
-
|
|
6286
|
-
|
|
6287
|
-
|
|
6288
|
-
|
|
6289
|
-
|
|
6290
|
-
|
|
6291
|
-
|
|
6292
|
-
|
|
6293
|
-
|
|
6294
|
-
|
|
6295
|
-
|
|
6296
|
-
|
|
6297
|
-
|
|
6298
|
-
|
|
6996
|
+
const gitOutcomes = await measureRuntimeProfileStage(
|
|
6997
|
+
deps.runtimeProfile,
|
|
6998
|
+
"efficiency.collect_git_outcomes",
|
|
6999
|
+
() => collectOutcomes({
|
|
7000
|
+
repoDir,
|
|
7001
|
+
granularity,
|
|
7002
|
+
timezone: dataset.normalizedInputs.timezone,
|
|
7003
|
+
since: options.since,
|
|
7004
|
+
until: options.until,
|
|
7005
|
+
includeMergeCommits: options.includeMergeCommits,
|
|
7006
|
+
activeUsageDays
|
|
7007
|
+
})
|
|
7008
|
+
);
|
|
7009
|
+
const repoScopedUsageRows = measureRuntimeProfileStageSync(
|
|
7010
|
+
deps.runtimeProfile,
|
|
7011
|
+
"efficiency.aggregate_usage",
|
|
7012
|
+
() => aggregateUsage(matchedEventsWithSignal, {
|
|
7013
|
+
granularity,
|
|
7014
|
+
timezone: dataset.normalizedInputs.timezone,
|
|
7015
|
+
includeModelBreakdown: false
|
|
7016
|
+
})
|
|
7017
|
+
);
|
|
7018
|
+
const rows = measureRuntimeProfileStageSync(
|
|
7019
|
+
deps.runtimeProfile,
|
|
7020
|
+
"efficiency.aggregate",
|
|
7021
|
+
() => aggregateEfficiency({
|
|
7022
|
+
usageRows: repoScopedUsageRows,
|
|
7023
|
+
periodOutcomes: gitOutcomes.periodOutcomes
|
|
7024
|
+
})
|
|
7025
|
+
);
|
|
7026
|
+
const usageDiagnostics = buildUsageDiagnostics({
|
|
7027
|
+
adaptersToParse: dataset.adaptersToParse,
|
|
7028
|
+
successfulParseResults: dataset.successfulParseResults,
|
|
7029
|
+
sourceFailures: dataset.sourceFailures,
|
|
7030
|
+
pricingOrigin,
|
|
7031
|
+
pricingWarning,
|
|
7032
|
+
activeEnvOverrides: dataset.readEnvVarOverrides(),
|
|
7033
|
+
timezone: dataset.normalizedInputs.timezone,
|
|
7034
|
+
runtimeProfile: deps.runtimeProfile?.snapshot()
|
|
6299
7035
|
});
|
|
6300
7036
|
return {
|
|
6301
7037
|
rows,
|
|
6302
7038
|
diagnostics: {
|
|
6303
|
-
usage:
|
|
7039
|
+
usage: usageDiagnostics,
|
|
6304
7040
|
repoDir: gitOutcomes.diagnostics.repoDir,
|
|
6305
7041
|
includeMergeCommits: gitOutcomes.diagnostics.includeMergeCommits,
|
|
6306
7042
|
gitCommitCount: gitOutcomes.diagnostics.commitsCollected,
|
|
@@ -6390,7 +7126,7 @@ function emitEnvVarOverrides(activeEnvOverrides, diagnosticsLogger) {
|
|
|
6390
7126
|
}
|
|
6391
7127
|
|
|
6392
7128
|
// src/cli/share-artifact.ts
|
|
6393
|
-
import { spawn as
|
|
7129
|
+
import { spawn as spawn4 } from "child_process";
|
|
6394
7130
|
import { writeFile as writeFile4 } from "fs/promises";
|
|
6395
7131
|
import path13 from "path";
|
|
6396
7132
|
async function writeShareSvgFile(fileName, svgContent) {
|
|
@@ -6424,7 +7160,7 @@ function resolveOpenCommand(filePath, platform) {
|
|
|
6424
7160
|
}
|
|
6425
7161
|
async function spawnDetached(command, args) {
|
|
6426
7162
|
await new Promise((resolve, reject) => {
|
|
6427
|
-
const child =
|
|
7163
|
+
const child = spawn4(command, args, {
|
|
6428
7164
|
detached: true,
|
|
6429
7165
|
stdio: "ignore",
|
|
6430
7166
|
windowsHide: true
|
|
@@ -6519,12 +7255,22 @@ async function prepareReport(options) {
|
|
|
6519
7255
|
validateOutputFormatOptions(options.commandOptions);
|
|
6520
7256
|
options.validate?.();
|
|
6521
7257
|
const format = resolveReportFormat(options.commandOptions, options.supportedFormats);
|
|
6522
|
-
const data = await
|
|
7258
|
+
const data = await measureRuntimeProfileStage(
|
|
7259
|
+
options.runtimeProfile,
|
|
7260
|
+
"report.prepare.build_data",
|
|
7261
|
+
options.buildData
|
|
7262
|
+
);
|
|
7263
|
+
const output = measureRuntimeProfileStageSync(
|
|
7264
|
+
options.runtimeProfile,
|
|
7265
|
+
"report.prepare.render",
|
|
7266
|
+
() => options.render(data, format)
|
|
7267
|
+
);
|
|
6523
7268
|
return {
|
|
6524
7269
|
format,
|
|
6525
7270
|
diagnostics: options.getDiagnostics(data),
|
|
6526
|
-
output
|
|
6527
|
-
shareArtifact: options.createShareArtifact?.(data)
|
|
7271
|
+
output,
|
|
7272
|
+
shareArtifact: options.createShareArtifact?.(data),
|
|
7273
|
+
runtimeProfile: options.runtimeProfile
|
|
6528
7274
|
};
|
|
6529
7275
|
}
|
|
6530
7276
|
async function writeShareArtifact(artifact) {
|
|
@@ -6543,6 +7289,13 @@ async function runPreparedReport(options) {
|
|
|
6543
7289
|
const envVarOverrides = options.getEnvVarOverrides?.(options.preparedReport.diagnostics) ?? [];
|
|
6544
7290
|
emitEnvVarOverrides(envVarOverrides, logger);
|
|
6545
7291
|
options.emitReportDiagnostics?.(options.preparedReport.diagnostics);
|
|
7292
|
+
emitRuntimeProfile(
|
|
7293
|
+
mergeRuntimeProfiles(
|
|
7294
|
+
options.preparedReport.runtimeProfile?.snapshot(),
|
|
7295
|
+
options.getRuntimeProfile?.(options.preparedReport.diagnostics)
|
|
7296
|
+
),
|
|
7297
|
+
logger
|
|
7298
|
+
);
|
|
6546
7299
|
if (options.warnOnTerminalOverflow && options.preparedReport.format === "terminal") {
|
|
6547
7300
|
warnIfTerminalTableOverflows(options.preparedReport.output, (message) => {
|
|
6548
7301
|
logger.warn(message);
|
|
@@ -6568,15 +7321,16 @@ function validateShareOption(granularity, options) {
|
|
|
6568
7321
|
throw new Error("--share is only supported for efficiency monthly");
|
|
6569
7322
|
}
|
|
6570
7323
|
}
|
|
6571
|
-
async function prepareEfficiencyReport(granularity, options) {
|
|
7324
|
+
async function prepareEfficiencyReport(granularity, options, deps = {}) {
|
|
6572
7325
|
return prepareReport({
|
|
6573
7326
|
commandOptions: options,
|
|
6574
7327
|
supportedFormats: efficiencyReportFormats,
|
|
6575
7328
|
validate: () => {
|
|
6576
7329
|
validateShareOption(granularity, options);
|
|
6577
7330
|
},
|
|
6578
|
-
buildData: () => buildEfficiencyData(granularity, options),
|
|
7331
|
+
buildData: () => buildEfficiencyData(granularity, options, deps),
|
|
6579
7332
|
getDiagnostics: (efficiencyData) => efficiencyData.diagnostics,
|
|
7333
|
+
runtimeProfile: deps.runtimeProfile,
|
|
6580
7334
|
createShareArtifact: options.share ? (efficiencyData) => ({
|
|
6581
7335
|
fileName: "efficiency-monthly-share.svg",
|
|
6582
7336
|
svg: renderEfficiencyMonthlyShareSvg(efficiencyData),
|
|
@@ -6600,7 +7354,8 @@ function emitEfficiencyReportDiagnostics(diagnostics) {
|
|
|
6600
7354
|
}
|
|
6601
7355
|
}
|
|
6602
7356
|
async function runEfficiencyReport(granularity, options) {
|
|
6603
|
-
const
|
|
7357
|
+
const runtimeProfile = createRuntimeProfileCollector();
|
|
7358
|
+
const preparedReport = await prepareEfficiencyReport(granularity, options, { runtimeProfile });
|
|
6604
7359
|
await runPreparedReport({
|
|
6605
7360
|
preparedReport,
|
|
6606
7361
|
emitCommonDiagnostics: (diagnostics) => {
|
|
@@ -6608,6 +7363,7 @@ async function runEfficiencyReport(granularity, options) {
|
|
|
6608
7363
|
},
|
|
6609
7364
|
getEnvVarOverrides: (diagnostics) => diagnostics.usage.activeEnvOverrides,
|
|
6610
7365
|
emitReportDiagnostics: emitEfficiencyReportDiagnostics,
|
|
7366
|
+
getRuntimeProfile: (diagnostics) => diagnostics.usage.runtimeProfile,
|
|
6611
7367
|
warnOnTerminalOverflow: true
|
|
6612
7368
|
});
|
|
6613
7369
|
}
|
|
@@ -6977,8 +7733,11 @@ var USD_PRECISION_SCALE3 = 1e12;
|
|
|
6977
7733
|
function roundUsd(value) {
|
|
6978
7734
|
return Math.round(value * USD_PRECISION_SCALE3) / USD_PRECISION_SCALE3;
|
|
6979
7735
|
}
|
|
6980
|
-
function
|
|
6981
|
-
return
|
|
7736
|
+
function addUsd3(left, right) {
|
|
7737
|
+
return roundUsd(left + right);
|
|
7738
|
+
}
|
|
7739
|
+
function hasAnyUsageSignal(period) {
|
|
7740
|
+
return period.totalTokens > 0 || period.baselineCostIncomplete || (period.baselineCostUsd ?? 0) > 0;
|
|
6982
7741
|
}
|
|
6983
7742
|
function parseCandidateModelsRaw(candidateModel) {
|
|
6984
7743
|
if (!candidateModel || Array.isArray(candidateModel) && candidateModel.length === 0) {
|
|
@@ -7044,30 +7803,41 @@ function withNotes(notes) {
|
|
|
7044
7803
|
}
|
|
7045
7804
|
function evaluateCandidateForPeriod(period, provider, candidateModel, pricingSource) {
|
|
7046
7805
|
const notes = /* @__PURE__ */ new Set();
|
|
7047
|
-
const
|
|
7806
|
+
const hasBillableUsage = hasBillableTokenBuckets(period);
|
|
7048
7807
|
const candidateResolvedModel = pricingSource ? pricingSource.resolveModelAlias(candidateModel) : candidateModel;
|
|
7049
7808
|
const pricing = pricingSource ? pricingSource.getPricing(candidateResolvedModel) : void 0;
|
|
7809
|
+
const syntheticEvent = createSyntheticEvent(period);
|
|
7050
7810
|
let hypotheticalCostUsd;
|
|
7051
7811
|
let hypotheticalCostIncomplete = false;
|
|
7052
|
-
if (!
|
|
7053
|
-
if (
|
|
7812
|
+
if (!hasBillableUsage) {
|
|
7813
|
+
if (!hasAnyUsageSignal(period)) {
|
|
7054
7814
|
hypotheticalCostUsd = 0;
|
|
7055
7815
|
} else {
|
|
7056
7816
|
hypotheticalCostUsd = void 0;
|
|
7057
7817
|
hypotheticalCostIncomplete = true;
|
|
7058
|
-
notes.add("
|
|
7059
|
-
}
|
|
7818
|
+
notes.add("usage_buckets_missing");
|
|
7819
|
+
}
|
|
7820
|
+
} else if (!pricing) {
|
|
7821
|
+
hypotheticalCostUsd = void 0;
|
|
7822
|
+
hypotheticalCostIncomplete = true;
|
|
7823
|
+
notes.add("missing_pricing");
|
|
7824
|
+
} else if (hasUsedBucketWithUndefinedRate(syntheticEvent, pricing)) {
|
|
7825
|
+
hypotheticalCostUsd = void 0;
|
|
7826
|
+
hypotheticalCostIncomplete = true;
|
|
7827
|
+
notes.add("missing_pricing");
|
|
7828
|
+
} else if (!canEstimateUsageCost(syntheticEvent, pricing)) {
|
|
7829
|
+
hypotheticalCostUsd = void 0;
|
|
7830
|
+
hypotheticalCostIncomplete = true;
|
|
7831
|
+
notes.add("usage_buckets_missing");
|
|
7060
7832
|
} else {
|
|
7061
|
-
hypotheticalCostUsd = roundUsd(
|
|
7062
|
-
calculateEstimatedCostUsd(createSyntheticEvent(period), pricing)
|
|
7063
|
-
);
|
|
7833
|
+
hypotheticalCostUsd = roundUsd(calculateEstimatedCostUsd(syntheticEvent, pricing));
|
|
7064
7834
|
}
|
|
7065
7835
|
let savingsUsd;
|
|
7066
7836
|
let savingsPct;
|
|
7067
7837
|
let hasBaselineTokenMismatch = false;
|
|
7068
7838
|
if (period.baselineCostIncomplete || period.baselineCostUsd === void 0) {
|
|
7069
7839
|
notes.add("baseline_incomplete");
|
|
7070
|
-
} else if (
|
|
7840
|
+
} else if (!hasBillableUsage && period.baselineCostUsd > 0) {
|
|
7071
7841
|
notes.add("baseline_tokens_missing");
|
|
7072
7842
|
hasBaselineTokenMismatch = true;
|
|
7073
7843
|
} else if (hypotheticalCostUsd !== void 0) {
|
|
@@ -7123,9 +7893,22 @@ function resolveBaselinePeriods(usageRows) {
|
|
|
7123
7893
|
periodRows.set(row.periodKey, row);
|
|
7124
7894
|
continue;
|
|
7125
7895
|
}
|
|
7126
|
-
|
|
7896
|
+
const existingRow = periodRows.get(row.periodKey);
|
|
7897
|
+
if (!existingRow) {
|
|
7127
7898
|
periodRows.set(row.periodKey, row);
|
|
7899
|
+
continue;
|
|
7128
7900
|
}
|
|
7901
|
+
periodRows.set(row.periodKey, {
|
|
7902
|
+
...existingRow,
|
|
7903
|
+
inputTokens: existingRow.inputTokens + row.inputTokens,
|
|
7904
|
+
outputTokens: existingRow.outputTokens + row.outputTokens,
|
|
7905
|
+
reasoningTokens: existingRow.reasoningTokens + row.reasoningTokens,
|
|
7906
|
+
cacheReadTokens: existingRow.cacheReadTokens + row.cacheReadTokens,
|
|
7907
|
+
cacheWriteTokens: existingRow.cacheWriteTokens + row.cacheWriteTokens,
|
|
7908
|
+
totalTokens: existingRow.totalTokens + row.totalTokens,
|
|
7909
|
+
costUsd: row.costUsd !== void 0 ? addUsd3(existingRow.costUsd ?? 0, row.costUsd) : existingRow.costUsd,
|
|
7910
|
+
costIncomplete: existingRow.costIncomplete === true || row.costIncomplete === true ? true : void 0
|
|
7911
|
+
});
|
|
7129
7912
|
}
|
|
7130
7913
|
const sortedPeriodKeys = [...periodRows.keys()].sort(compareByCodePoint);
|
|
7131
7914
|
const periods = sortedPeriodKeys.map((periodKey) => {
|
|
@@ -7225,16 +8008,13 @@ function buildCounterfactualRows(input) {
|
|
|
7225
8008
|
|
|
7226
8009
|
// src/cli/build-optimize-data.ts
|
|
7227
8010
|
function resolveOptimizeProvider(providers, providerFilter) {
|
|
7228
|
-
const distinctProviders =
|
|
8011
|
+
const distinctProviders = collectCanonicalProviderRoots(providers);
|
|
7229
8012
|
const normalizedProviderFilter = normalizeProviderFilter(providerFilter);
|
|
7230
8013
|
if (distinctProviders.length > 1) {
|
|
7231
8014
|
if (normalizedProviderFilter) {
|
|
7232
8015
|
const matchingProviders = distinctProviders.filter(
|
|
7233
|
-
(provider) => provider
|
|
8016
|
+
(provider) => matchesCanonicalProviderFilter(provider, normalizedProviderFilter)
|
|
7234
8017
|
);
|
|
7235
|
-
if (matchingProviders.includes(normalizedProviderFilter)) {
|
|
7236
|
-
return normalizedProviderFilter;
|
|
7237
|
-
}
|
|
7238
8018
|
if (matchingProviders.length === 1) {
|
|
7239
8019
|
return matchingProviders[0];
|
|
7240
8020
|
}
|
|
@@ -7261,28 +8041,41 @@ function resolveOptimizeProvider(providers, providerFilter) {
|
|
|
7261
8041
|
async function buildOptimizeData(granularity, options, deps = {}) {
|
|
7262
8042
|
const candidateModels = normalizeCandidateModels(options.candidateModel);
|
|
7263
8043
|
const top = parseTopOption(options.top);
|
|
7264
|
-
const dataset = await
|
|
8044
|
+
const dataset = await measureRuntimeProfileStage(
|
|
8045
|
+
deps.runtimeProfile,
|
|
8046
|
+
"optimize.dataset.total",
|
|
8047
|
+
() => buildUsageEventDataset(options, deps)
|
|
8048
|
+
);
|
|
7265
8049
|
const detectedProviders = new Set(
|
|
7266
8050
|
dataset.filteredEvents.map((event) => normalizeProviderFilter(event.provider)).filter((provider2) => provider2 !== void 0)
|
|
7267
8051
|
);
|
|
7268
|
-
const provider =
|
|
7269
|
-
|
|
7270
|
-
|
|
8052
|
+
const provider = measureRuntimeProfileStageSync(
|
|
8053
|
+
deps.runtimeProfile,
|
|
8054
|
+
"optimize.resolve_provider",
|
|
8055
|
+
() => resolveOptimizeProvider(detectedProviders, dataset.normalizedInputs.providerFilter)
|
|
7271
8056
|
);
|
|
7272
8057
|
const { pricedEvents, pricingOrigin, pricingWarning, pricingSource } = await applyPricingToUsageEventDataset(dataset, deps, "force");
|
|
7273
|
-
const usageRows =
|
|
7274
|
-
|
|
7275
|
-
|
|
7276
|
-
|
|
7277
|
-
|
|
7278
|
-
|
|
7279
|
-
|
|
7280
|
-
|
|
7281
|
-
|
|
7282
|
-
|
|
7283
|
-
|
|
7284
|
-
|
|
7285
|
-
|
|
8058
|
+
const usageRows = measureRuntimeProfileStageSync(
|
|
8059
|
+
deps.runtimeProfile,
|
|
8060
|
+
"optimize.aggregate_usage",
|
|
8061
|
+
() => aggregateUsage(pricedEvents, {
|
|
8062
|
+
granularity,
|
|
8063
|
+
timezone: dataset.normalizedInputs.timezone,
|
|
8064
|
+
sourceOrder: dataset.adaptersToParse.map((adapter) => adapter.id),
|
|
8065
|
+
includeModelBreakdown: false
|
|
8066
|
+
})
|
|
8067
|
+
);
|
|
8068
|
+
const counterfactual = measureRuntimeProfileStageSync(
|
|
8069
|
+
deps.runtimeProfile,
|
|
8070
|
+
"optimize.counterfactual",
|
|
8071
|
+
() => buildCounterfactualRows({
|
|
8072
|
+
usageRows,
|
|
8073
|
+
provider,
|
|
8074
|
+
candidateModels,
|
|
8075
|
+
pricingSource,
|
|
8076
|
+
top
|
|
8077
|
+
})
|
|
8078
|
+
);
|
|
7286
8079
|
const usageDiagnostics = buildUsageDiagnostics({
|
|
7287
8080
|
adaptersToParse: dataset.adaptersToParse,
|
|
7288
8081
|
successfulParseResults: dataset.successfulParseResults,
|
|
@@ -7290,7 +8083,8 @@ async function buildOptimizeData(granularity, options, deps = {}) {
|
|
|
7290
8083
|
pricingOrigin,
|
|
7291
8084
|
pricingWarning,
|
|
7292
8085
|
activeEnvOverrides: dataset.readEnvVarOverrides(),
|
|
7293
|
-
timezone: dataset.normalizedInputs.timezone
|
|
8086
|
+
timezone: dataset.normalizedInputs.timezone,
|
|
8087
|
+
runtimeProfile: deps.runtimeProfile?.snapshot()
|
|
7294
8088
|
});
|
|
7295
8089
|
return {
|
|
7296
8090
|
rows: counterfactual.rows,
|
|
@@ -7318,7 +8112,7 @@ function validateShareOption2(granularity, options) {
|
|
|
7318
8112
|
throw new Error("--share is only supported for optimize monthly");
|
|
7319
8113
|
}
|
|
7320
8114
|
}
|
|
7321
|
-
async function prepareOptimizeReport(granularity, options) {
|
|
8115
|
+
async function prepareOptimizeReport(granularity, options, deps = {}) {
|
|
7322
8116
|
return prepareReport({
|
|
7323
8117
|
commandOptions: options,
|
|
7324
8118
|
supportedFormats: optimizeReportFormats,
|
|
@@ -7326,7 +8120,7 @@ async function prepareOptimizeReport(granularity, options) {
|
|
|
7326
8120
|
validateShareOption2(granularity, options);
|
|
7327
8121
|
},
|
|
7328
8122
|
buildData: async () => {
|
|
7329
|
-
const optimizeData = await buildOptimizeData(granularity, options);
|
|
8123
|
+
const optimizeData = await buildOptimizeData(granularity, options, deps);
|
|
7330
8124
|
return {
|
|
7331
8125
|
optimizeData,
|
|
7332
8126
|
candidateCount: optimizeData.rows.filter(
|
|
@@ -7338,6 +8132,7 @@ async function prepareOptimizeReport(granularity, options) {
|
|
|
7338
8132
|
...bundle.optimizeData.diagnostics,
|
|
7339
8133
|
candidateCount: bundle.candidateCount
|
|
7340
8134
|
}),
|
|
8135
|
+
runtimeProfile: deps.runtimeProfile,
|
|
7341
8136
|
createShareArtifact: options.share ? (bundle) => ({
|
|
7342
8137
|
fileName: "optimize-monthly-share.svg",
|
|
7343
8138
|
svg: renderOptimizeMonthlyShareSvg(bundle.optimizeData),
|
|
@@ -7362,7 +8157,8 @@ function emitOptimizeReportDiagnostics(diagnostics) {
|
|
|
7362
8157
|
}
|
|
7363
8158
|
}
|
|
7364
8159
|
async function runOptimizeReport(granularity, options) {
|
|
7365
|
-
const
|
|
8160
|
+
const runtimeProfile = createRuntimeProfileCollector();
|
|
8161
|
+
const preparedReport = await prepareOptimizeReport(granularity, options, { runtimeProfile });
|
|
7366
8162
|
await runPreparedReport({
|
|
7367
8163
|
preparedReport,
|
|
7368
8164
|
emitCommonDiagnostics: (diagnostics) => {
|
|
@@ -7370,6 +8166,7 @@ async function runOptimizeReport(granularity, options) {
|
|
|
7370
8166
|
},
|
|
7371
8167
|
getEnvVarOverrides: (diagnostics) => diagnostics.usage.activeEnvOverrides,
|
|
7372
8168
|
emitReportDiagnostics: emitOptimizeReportDiagnostics,
|
|
8169
|
+
getRuntimeProfile: (diagnostics) => diagnostics.usage.runtimeProfile,
|
|
7373
8170
|
warnOnTerminalOverflow: true
|
|
7374
8171
|
});
|
|
7375
8172
|
}
|
|
@@ -7387,6 +8184,7 @@ var compactNumberFormatter = new Intl.NumberFormat("en-US", {
|
|
|
7387
8184
|
maximumFractionDigits: 1
|
|
7388
8185
|
});
|
|
7389
8186
|
var sparklineBlocks = [" ", "\u2581", "\u2582", "\u2583", "\u2584", "\u2585", "\u2586", "\u2587", "\u2588"];
|
|
8187
|
+
var minimumCompressedEmptyEdgeDays = 7;
|
|
7390
8188
|
function resolveTerminalWidth2(override) {
|
|
7391
8189
|
if (typeof override === "number" && Number.isFinite(override) && override > 0) {
|
|
7392
8190
|
return Math.floor(override);
|
|
@@ -7426,15 +8224,26 @@ function downsampleBuckets(buckets, maxColumns) {
|
|
|
7426
8224
|
const startIndex = Math.floor(columnIndex * buckets.length / maxColumns);
|
|
7427
8225
|
const endIndex = Math.floor((columnIndex + 1) * buckets.length / maxColumns);
|
|
7428
8226
|
const slice = buckets.slice(startIndex, Math.max(startIndex + 1, endIndex));
|
|
7429
|
-
const
|
|
8227
|
+
const peakValue = slice.reduce((maxValue, bucket) => Math.max(maxValue, bucket.value), 0);
|
|
7430
8228
|
return {
|
|
7431
8229
|
date: slice[0]?.date ?? "",
|
|
7432
|
-
value:
|
|
8230
|
+
value: peakValue,
|
|
7433
8231
|
observed: slice.some((bucket) => bucket.observed),
|
|
7434
8232
|
incomplete: slice.some((bucket) => bucket.incomplete === true) || void 0
|
|
7435
8233
|
};
|
|
7436
8234
|
});
|
|
7437
8235
|
}
|
|
8236
|
+
function getBarLevel(value, maxValue, rowCount) {
|
|
8237
|
+
if (value <= 0 || maxValue <= 0 || rowCount <= 0) {
|
|
8238
|
+
return 0;
|
|
8239
|
+
}
|
|
8240
|
+
return Math.max(1, Math.min(rowCount * 8, Math.round(value / maxValue * rowCount * 8)));
|
|
8241
|
+
}
|
|
8242
|
+
function renderCombinedChartCell(barLevel, rowIndex, rowCount) {
|
|
8243
|
+
const rowBaseLevel = (rowCount - rowIndex - 1) * 8;
|
|
8244
|
+
const cellLevel = Math.max(0, Math.min(8, barLevel - rowBaseLevel));
|
|
8245
|
+
return sparklineBlocks[cellLevel];
|
|
8246
|
+
}
|
|
7438
8247
|
function isApproximatePeak(series, metric) {
|
|
7439
8248
|
if (metric !== "cost" || series.summary.observedDayCount === 0) {
|
|
7440
8249
|
return false;
|
|
@@ -7483,18 +8292,88 @@ function renderSummaryOnly(trendsData, options) {
|
|
|
7483
8292
|
lines.push(renderSummary(trendsData.totalSeries, trendsData.metric));
|
|
7484
8293
|
return lines.join("\n");
|
|
7485
8294
|
}
|
|
8295
|
+
function isInactiveEdgeBucket(bucket) {
|
|
8296
|
+
return !bucket.observed && bucket.value === 0 && bucket.incomplete !== true;
|
|
8297
|
+
}
|
|
8298
|
+
function findLeadingInactiveBucketCount(buckets) {
|
|
8299
|
+
let count = 0;
|
|
8300
|
+
while (count < buckets.length && isInactiveEdgeBucket(buckets[count])) {
|
|
8301
|
+
count += 1;
|
|
8302
|
+
}
|
|
8303
|
+
return count;
|
|
8304
|
+
}
|
|
8305
|
+
function findTrailingInactiveBucketCount(buckets) {
|
|
8306
|
+
let count = 0;
|
|
8307
|
+
while (count < buckets.length && isInactiveEdgeBucket(buckets[buckets.length - 1 - count])) {
|
|
8308
|
+
count += 1;
|
|
8309
|
+
}
|
|
8310
|
+
return count;
|
|
8311
|
+
}
|
|
8312
|
+
function formatCompressedEdgeNote(count, startDate, endDate, position) {
|
|
8313
|
+
const rangeLabel = startDate === endDate ? formatDateLabel(startDate) : `${formatDateLabel(startDate)}-${formatDateLabel(endDate)}`;
|
|
8314
|
+
const activityLabel = position === "before" ? "before first activity" : "after last activity";
|
|
8315
|
+
return `Compressed ${count} empty day(s) ${activityLabel} (${rangeLabel}).`;
|
|
8316
|
+
}
|
|
8317
|
+
function getCombinedChartView(series) {
|
|
8318
|
+
if (series.buckets.length === 0) {
|
|
8319
|
+
return {
|
|
8320
|
+
chartSeries: series,
|
|
8321
|
+
notes: []
|
|
8322
|
+
};
|
|
8323
|
+
}
|
|
8324
|
+
const leadingInactiveBucketCount = findLeadingInactiveBucketCount(series.buckets);
|
|
8325
|
+
const trailingInactiveBucketCount = findTrailingInactiveBucketCount(series.buckets);
|
|
8326
|
+
const notes = [];
|
|
8327
|
+
let startIndex = 0;
|
|
8328
|
+
let endIndex = series.buckets.length - 1;
|
|
8329
|
+
if (leadingInactiveBucketCount >= minimumCompressedEmptyEdgeDays && leadingInactiveBucketCount < series.buckets.length) {
|
|
8330
|
+
const firstCompressedBucket = series.buckets[0];
|
|
8331
|
+
const lastCompressedBucket = series.buckets[leadingInactiveBucketCount - 1];
|
|
8332
|
+
notes.push(
|
|
8333
|
+
formatCompressedEdgeNote(
|
|
8334
|
+
leadingInactiveBucketCount,
|
|
8335
|
+
firstCompressedBucket.date,
|
|
8336
|
+
lastCompressedBucket.date,
|
|
8337
|
+
"before"
|
|
8338
|
+
)
|
|
8339
|
+
);
|
|
8340
|
+
startIndex = leadingInactiveBucketCount;
|
|
8341
|
+
}
|
|
8342
|
+
if (trailingInactiveBucketCount >= minimumCompressedEmptyEdgeDays && trailingInactiveBucketCount < series.buckets.length - startIndex) {
|
|
8343
|
+
const firstCompressedBucket = series.buckets[series.buckets.length - trailingInactiveBucketCount];
|
|
8344
|
+
const lastCompressedBucket = series.buckets[series.buckets.length - 1];
|
|
8345
|
+
notes.push(
|
|
8346
|
+
formatCompressedEdgeNote(
|
|
8347
|
+
trailingInactiveBucketCount,
|
|
8348
|
+
firstCompressedBucket.date,
|
|
8349
|
+
lastCompressedBucket.date,
|
|
8350
|
+
"after"
|
|
8351
|
+
)
|
|
8352
|
+
);
|
|
8353
|
+
endIndex = series.buckets.length - trailingInactiveBucketCount - 1;
|
|
8354
|
+
}
|
|
8355
|
+
return {
|
|
8356
|
+
chartSeries: notes.length === 0 ? series : {
|
|
8357
|
+
...series,
|
|
8358
|
+
buckets: series.buckets.slice(startIndex, endIndex + 1)
|
|
8359
|
+
},
|
|
8360
|
+
notes
|
|
8361
|
+
};
|
|
8362
|
+
}
|
|
7486
8363
|
function renderCombinedChart(series, metric, plotWidth) {
|
|
7487
8364
|
const buckets = downsampleBuckets(series.buckets, plotWidth);
|
|
7488
8365
|
const maxValue = Math.max(...buckets.map((bucket) => bucket.value), 0);
|
|
7489
8366
|
const lines = [];
|
|
7490
|
-
const
|
|
7491
|
-
|
|
7492
|
-
|
|
8367
|
+
const chartRowCount = 4;
|
|
8368
|
+
const tickValues = Array.from({ length: chartRowCount + 1 }, (_, index) => {
|
|
8369
|
+
const inverseIndex = chartRowCount - index;
|
|
8370
|
+
return maxValue === 0 ? 0 : maxValue * inverseIndex / chartRowCount;
|
|
7493
8371
|
});
|
|
7494
8372
|
const labelWidth = tickValues.reduce(
|
|
7495
8373
|
(maxWidth, value) => Math.max(maxWidth, visibleWidth(formatAxisValue(value, metric))),
|
|
7496
8374
|
0
|
|
7497
8375
|
);
|
|
8376
|
+
const barLevels = buckets.map((bucket) => getBarLevel(bucket.value, maxValue, chartRowCount));
|
|
7498
8377
|
tickValues.forEach((tickValue, tickIndex) => {
|
|
7499
8378
|
if (tickIndex === tickValues.length - 1) {
|
|
7500
8379
|
lines.push(
|
|
@@ -7502,18 +8381,7 @@ function renderCombinedChart(series, metric, plotWidth) {
|
|
|
7502
8381
|
);
|
|
7503
8382
|
return;
|
|
7504
8383
|
}
|
|
7505
|
-
const
|
|
7506
|
-
const glyphs = buckets.map((bucket) => {
|
|
7507
|
-
if (maxValue === 0) {
|
|
7508
|
-
return " ";
|
|
7509
|
-
}
|
|
7510
|
-
const ratio = bucket.value / maxValue;
|
|
7511
|
-
const level = Math.max(0, Math.min(8, Math.round(ratio * 8)));
|
|
7512
|
-
if (bucket.value >= threshold && level > 0) {
|
|
7513
|
-
return sparklineBlocks[level];
|
|
7514
|
-
}
|
|
7515
|
-
return " ";
|
|
7516
|
-
}).join("");
|
|
8384
|
+
const glyphs = barLevels.map((barLevel) => renderCombinedChartCell(barLevel, tickIndex, chartRowCount)).join("");
|
|
7517
8385
|
lines.push(`${formatAxisValue(tickValue, metric).padStart(labelWidth)} \u2524${glyphs}`);
|
|
7518
8386
|
});
|
|
7519
8387
|
const startLabel = formatDateLabel(series.buckets[0]?.date ?? "");
|
|
@@ -7597,12 +8465,14 @@ function renderTerminalTrendsReport(trendsData, options) {
|
|
|
7597
8465
|
...renderSourceLines(trendsData.sourceSeries, trendsData.metric, terminalWidth ?? 80)
|
|
7598
8466
|
);
|
|
7599
8467
|
} else {
|
|
8468
|
+
const chartView = getCombinedChartView(trendsData.totalSeries);
|
|
8469
|
+
if (chartView.notes.length > 0) {
|
|
8470
|
+
lines.push(...chartView.notes);
|
|
8471
|
+
}
|
|
7600
8472
|
const plotWidth = Math.max(16, (terminalWidth ?? 80) - 14);
|
|
7601
|
-
const chartLines = renderCombinedChart(
|
|
7602
|
-
|
|
7603
|
-
|
|
7604
|
-
plotWidth
|
|
7605
|
-
).map((line) => useColor ? accent(line) : line);
|
|
8473
|
+
const chartLines = renderCombinedChart(chartView.chartSeries, trendsData.metric, plotWidth).map(
|
|
8474
|
+
(line) => useColor ? accent(line) : line
|
|
8475
|
+
);
|
|
7606
8476
|
lines.push(...chartLines);
|
|
7607
8477
|
}
|
|
7608
8478
|
lines.push("");
|
|
@@ -7628,6 +8498,13 @@ function renderTrendsReport(trendsData, format, options = {}) {
|
|
|
7628
8498
|
}
|
|
7629
8499
|
|
|
7630
8500
|
// src/trends/aggregate-trends.ts
|
|
8501
|
+
var VALUE_PRECISION_SCALE = 1e12;
|
|
8502
|
+
function addValue(left, right) {
|
|
8503
|
+
return Math.round((left + right) * VALUE_PRECISION_SCALE) / VALUE_PRECISION_SCALE;
|
|
8504
|
+
}
|
|
8505
|
+
function divideValue(value, divisor) {
|
|
8506
|
+
return Math.round(value / divisor * VALUE_PRECISION_SCALE) / VALUE_PRECISION_SCALE;
|
|
8507
|
+
}
|
|
7631
8508
|
function toTrendBucket(row, metric) {
|
|
7632
8509
|
return {
|
|
7633
8510
|
date: row.periodKey,
|
|
@@ -7656,12 +8533,12 @@ function buildTrendSummary(buckets) {
|
|
|
7656
8533
|
observedDayCount: 0
|
|
7657
8534
|
};
|
|
7658
8535
|
}
|
|
7659
|
-
const total = buckets.reduce((sum, bucket) => sum
|
|
8536
|
+
const total = buckets.reduce((sum, bucket) => addValue(sum, bucket.value), 0);
|
|
7660
8537
|
const observedBuckets = buckets.filter((bucket) => bucket.observed);
|
|
7661
8538
|
if (observedBuckets.length === 0) {
|
|
7662
8539
|
return {
|
|
7663
8540
|
total,
|
|
7664
|
-
average: buckets.length > 0 ? total
|
|
8541
|
+
average: buckets.length > 0 ? divideValue(total, buckets.length) : 0,
|
|
7665
8542
|
peak: {
|
|
7666
8543
|
date: "",
|
|
7667
8544
|
value: 0
|
|
@@ -7677,7 +8554,7 @@ function buildTrendSummary(buckets) {
|
|
|
7677
8554
|
);
|
|
7678
8555
|
return {
|
|
7679
8556
|
total,
|
|
7680
|
-
average: buckets.length > 0 ? total
|
|
8557
|
+
average: buckets.length > 0 ? divideValue(total, buckets.length) : 0,
|
|
7681
8558
|
peak: {
|
|
7682
8559
|
date: peak.date,
|
|
7683
8560
|
value: peak.value
|
|
@@ -7697,6 +8574,46 @@ function buildSeries(source, rowsByDate, dateKeys, metric) {
|
|
|
7697
8574
|
summary: buildTrendSummary(buckets)
|
|
7698
8575
|
};
|
|
7699
8576
|
}
|
|
8577
|
+
function createEmptyUsageRow(periodKey, rowType, source) {
|
|
8578
|
+
return rowType === "period_combined" ? {
|
|
8579
|
+
rowType,
|
|
8580
|
+
periodKey,
|
|
8581
|
+
source: "combined",
|
|
8582
|
+
models: [],
|
|
8583
|
+
modelBreakdown: [],
|
|
8584
|
+
inputTokens: 0,
|
|
8585
|
+
outputTokens: 0,
|
|
8586
|
+
reasoningTokens: 0,
|
|
8587
|
+
cacheReadTokens: 0,
|
|
8588
|
+
cacheWriteTokens: 0,
|
|
8589
|
+
totalTokens: 0
|
|
8590
|
+
} : {
|
|
8591
|
+
rowType,
|
|
8592
|
+
periodKey,
|
|
8593
|
+
source,
|
|
8594
|
+
models: [],
|
|
8595
|
+
modelBreakdown: [],
|
|
8596
|
+
inputTokens: 0,
|
|
8597
|
+
outputTokens: 0,
|
|
8598
|
+
reasoningTokens: 0,
|
|
8599
|
+
cacheReadTokens: 0,
|
|
8600
|
+
cacheWriteTokens: 0,
|
|
8601
|
+
totalTokens: 0
|
|
8602
|
+
};
|
|
8603
|
+
}
|
|
8604
|
+
function addRowTotals(target, row) {
|
|
8605
|
+
return {
|
|
8606
|
+
...target,
|
|
8607
|
+
inputTokens: target.inputTokens + row.inputTokens,
|
|
8608
|
+
outputTokens: target.outputTokens + row.outputTokens,
|
|
8609
|
+
reasoningTokens: target.reasoningTokens + row.reasoningTokens,
|
|
8610
|
+
cacheReadTokens: target.cacheReadTokens + row.cacheReadTokens,
|
|
8611
|
+
cacheWriteTokens: target.cacheWriteTokens + row.cacheWriteTokens,
|
|
8612
|
+
totalTokens: target.totalTokens + row.totalTokens,
|
|
8613
|
+
costUsd: row.costUsd !== void 0 ? addValue(target.costUsd ?? 0, row.costUsd) : target.costUsd,
|
|
8614
|
+
costIncomplete: target.costIncomplete === true || row.costIncomplete === true ? true : void 0
|
|
8615
|
+
};
|
|
8616
|
+
}
|
|
7700
8617
|
function toCombinedRowsByDate(rows) {
|
|
7701
8618
|
const combinedByDate = /* @__PURE__ */ new Map();
|
|
7702
8619
|
const sourceOnlyByDate = /* @__PURE__ */ new Map();
|
|
@@ -7708,9 +8625,8 @@ function toCombinedRowsByDate(rows) {
|
|
|
7708
8625
|
combinedByDate.set(row.periodKey, row);
|
|
7709
8626
|
continue;
|
|
7710
8627
|
}
|
|
7711
|
-
|
|
7712
|
-
|
|
7713
|
-
}
|
|
8628
|
+
const existingSourceOnlyRow = sourceOnlyByDate.get(row.periodKey) ?? createEmptyUsageRow(row.periodKey, "period_combined", "combined");
|
|
8629
|
+
sourceOnlyByDate.set(row.periodKey, addRowTotals(existingSourceOnlyRow, row));
|
|
7714
8630
|
}
|
|
7715
8631
|
const resolved = /* @__PURE__ */ new Map();
|
|
7716
8632
|
for (const [date, row] of sourceOnlyByDate) {
|
|
@@ -7731,7 +8647,8 @@ function toSourceSeries(rows, dateKeys, options) {
|
|
|
7731
8647
|
continue;
|
|
7732
8648
|
}
|
|
7733
8649
|
const sourceRows = rowsBySource.get(row.source) ?? /* @__PURE__ */ new Map();
|
|
7734
|
-
sourceRows.
|
|
8650
|
+
const existingSourceRow = sourceRows.get(row.periodKey) ?? createEmptyUsageRow(row.periodKey, "period_source", row.source);
|
|
8651
|
+
sourceRows.set(row.periodKey, addRowTotals(existingSourceRow, row));
|
|
7735
8652
|
rowsBySource.set(row.source, sourceRows);
|
|
7736
8653
|
}
|
|
7737
8654
|
const observedSources = [...rowsBySource.keys()].sort((left, right) => {
|
|
@@ -7878,36 +8795,48 @@ async function buildTrendsData(options, deps = {}) {
|
|
|
7878
8795
|
const timezone = options.timezone?.trim() ?? Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
7879
8796
|
validateTimezone(timezone);
|
|
7880
8797
|
const resolved = resolveTrendsOptions(options, timezone, now);
|
|
7881
|
-
const dataset = await
|
|
7882
|
-
|
|
7883
|
-
|
|
7884
|
-
|
|
7885
|
-
|
|
7886
|
-
|
|
7887
|
-
|
|
7888
|
-
|
|
8798
|
+
const dataset = await measureRuntimeProfileStage(
|
|
8799
|
+
deps.runtimeProfile,
|
|
8800
|
+
"trends.dataset.total",
|
|
8801
|
+
() => buildUsageEventDataset(
|
|
8802
|
+
{
|
|
8803
|
+
...options,
|
|
8804
|
+
timezone,
|
|
8805
|
+
since: resolved.fetchDateRange?.from ?? options.since,
|
|
8806
|
+
until: resolved.fetchDateRange?.to ?? options.until
|
|
8807
|
+
},
|
|
8808
|
+
deps
|
|
8809
|
+
)
|
|
7889
8810
|
);
|
|
7890
8811
|
const pricingResult = resolved.metric === "cost" ? await applyPricingToUsageEventDataset(dataset, deps, "auto") : {
|
|
7891
8812
|
pricedEvents: dataset.filteredEvents,
|
|
7892
8813
|
pricingOrigin: "none",
|
|
7893
8814
|
pricingWarning: void 0
|
|
7894
8815
|
};
|
|
7895
|
-
const dailyRows =
|
|
7896
|
-
|
|
7897
|
-
|
|
7898
|
-
|
|
7899
|
-
|
|
7900
|
-
|
|
8816
|
+
const dailyRows = measureRuntimeProfileStageSync(
|
|
8817
|
+
deps.runtimeProfile,
|
|
8818
|
+
"trends.aggregate_usage",
|
|
8819
|
+
() => aggregateUsage(pricingResult.pricedEvents, {
|
|
8820
|
+
granularity: "daily",
|
|
8821
|
+
timezone: dataset.normalizedInputs.timezone,
|
|
8822
|
+
sourceOrder: dataset.adaptersToParse.map((adapter) => adapter.id),
|
|
8823
|
+
includeModelBreakdown: false
|
|
8824
|
+
})
|
|
8825
|
+
);
|
|
7901
8826
|
const observedDates = dailyRows.filter((row) => row.rowType !== "grand_total").map((row) => row.periodKey).sort();
|
|
7902
8827
|
const outputDateRange = resolveOutputDateRange(options, resolved.today, resolved.days, [
|
|
7903
8828
|
...new Set(observedDates)
|
|
7904
8829
|
]);
|
|
7905
|
-
const trends =
|
|
7906
|
-
|
|
7907
|
-
|
|
7908
|
-
|
|
7909
|
-
|
|
7910
|
-
|
|
8830
|
+
const trends = measureRuntimeProfileStageSync(
|
|
8831
|
+
deps.runtimeProfile,
|
|
8832
|
+
"trends.aggregate",
|
|
8833
|
+
() => aggregateTrends(filterRowsToDateRange(dailyRows, outputDateRange), {
|
|
8834
|
+
dateRange: outputDateRange,
|
|
8835
|
+
metric: resolved.metric,
|
|
8836
|
+
bySource: options.bySource === true,
|
|
8837
|
+
sourceOrder: dataset.adaptersToParse.map((adapter) => adapter.id)
|
|
8838
|
+
})
|
|
8839
|
+
);
|
|
7911
8840
|
const diagnostics = buildUsageDiagnostics({
|
|
7912
8841
|
adaptersToParse: dataset.adaptersToParse,
|
|
7913
8842
|
successfulParseResults: dataset.successfulParseResults,
|
|
@@ -7915,7 +8844,8 @@ async function buildTrendsData(options, deps = {}) {
|
|
|
7915
8844
|
pricingOrigin: pricingResult.pricingOrigin,
|
|
7916
8845
|
pricingWarning: pricingResult.pricingWarning,
|
|
7917
8846
|
activeEnvOverrides: dataset.readEnvVarOverrides(),
|
|
7918
|
-
timezone: dataset.normalizedInputs.timezone
|
|
8847
|
+
timezone: dataset.normalizedInputs.timezone,
|
|
8848
|
+
runtimeProfile: deps.runtimeProfile?.snapshot()
|
|
7919
8849
|
});
|
|
7920
8850
|
return {
|
|
7921
8851
|
metric: resolved.metric,
|
|
@@ -7928,22 +8858,59 @@ async function buildTrendsData(options, deps = {}) {
|
|
|
7928
8858
|
|
|
7929
8859
|
// src/cli/run-trends-report.ts
|
|
7930
8860
|
var trendsReportFormats = ["terminal", "json"];
|
|
7931
|
-
async function prepareTrendsReport(options) {
|
|
8861
|
+
async function prepareTrendsReport(options, deps = {}) {
|
|
7932
8862
|
return prepareReport({
|
|
7933
8863
|
commandOptions: options,
|
|
7934
8864
|
supportedFormats: trendsReportFormats,
|
|
7935
|
-
buildData: () => buildTrendsData(options),
|
|
8865
|
+
buildData: () => buildTrendsData(options, deps),
|
|
7936
8866
|
getDiagnostics: (trendsData) => trendsData.diagnostics,
|
|
8867
|
+
runtimeProfile: deps.runtimeProfile,
|
|
7937
8868
|
render: (trendsData, format) => renderTrendsReport(trendsData, format)
|
|
7938
8869
|
});
|
|
7939
8870
|
}
|
|
7940
8871
|
async function runTrendsReport(options) {
|
|
7941
|
-
const
|
|
8872
|
+
const runtimeProfile = createRuntimeProfileCollector();
|
|
8873
|
+
const preparedReport = await prepareTrendsReport(options, { runtimeProfile });
|
|
7942
8874
|
await runPreparedReport({
|
|
7943
8875
|
preparedReport,
|
|
7944
8876
|
emitCommonDiagnostics: emitDiagnostics,
|
|
7945
|
-
getEnvVarOverrides: (diagnostics) => diagnostics.activeEnvOverrides
|
|
8877
|
+
getEnvVarOverrides: (diagnostics) => diagnostics.activeEnvOverrides,
|
|
8878
|
+
getRuntimeProfile: (diagnostics) => diagnostics.runtimeProfile
|
|
8879
|
+
});
|
|
8880
|
+
}
|
|
8881
|
+
|
|
8882
|
+
// src/cli/build-usage-data.ts
|
|
8883
|
+
async function buildUsageData(granularity, options, deps = {}) {
|
|
8884
|
+
const dataset = await measureRuntimeProfileStage(
|
|
8885
|
+
deps.runtimeProfile,
|
|
8886
|
+
"usage.dataset.total",
|
|
8887
|
+
() => buildUsageEventDataset(options, deps)
|
|
8888
|
+
);
|
|
8889
|
+
const { pricedEvents, pricingOrigin, pricingWarning } = await applyPricingToUsageEventDataset(
|
|
8890
|
+
dataset,
|
|
8891
|
+
deps,
|
|
8892
|
+
"auto"
|
|
8893
|
+
);
|
|
8894
|
+
const rows = measureRuntimeProfileStageSync(
|
|
8895
|
+
deps.runtimeProfile,
|
|
8896
|
+
"usage.aggregate",
|
|
8897
|
+
() => aggregateUsage(pricedEvents, {
|
|
8898
|
+
granularity,
|
|
8899
|
+
timezone: dataset.normalizedInputs.timezone,
|
|
8900
|
+
sourceOrder: dataset.adaptersToParse.map((adapter) => adapter.id)
|
|
8901
|
+
})
|
|
8902
|
+
);
|
|
8903
|
+
const diagnostics = buildUsageDiagnostics({
|
|
8904
|
+
adaptersToParse: dataset.adaptersToParse,
|
|
8905
|
+
successfulParseResults: dataset.successfulParseResults,
|
|
8906
|
+
sourceFailures: dataset.sourceFailures,
|
|
8907
|
+
pricingOrigin,
|
|
8908
|
+
pricingWarning,
|
|
8909
|
+
activeEnvOverrides: dataset.readEnvVarOverrides(),
|
|
8910
|
+
timezone: dataset.normalizedInputs.timezone,
|
|
8911
|
+
runtimeProfile: deps.runtimeProfile?.snapshot()
|
|
7946
8912
|
});
|
|
8913
|
+
return assembleUsageDataResult(pricedEvents, rows, diagnostics);
|
|
7947
8914
|
}
|
|
7948
8915
|
|
|
7949
8916
|
// src/render/markdown-table.ts
|
|
@@ -8271,13 +9238,14 @@ function resolveTableLayout(options) {
|
|
|
8271
9238
|
function resolveShareFileName(granularity) {
|
|
8272
9239
|
return `usage-${granularity}-share.svg`;
|
|
8273
9240
|
}
|
|
8274
|
-
async function prepareUsageReport(granularity, options) {
|
|
9241
|
+
async function prepareUsageReport(granularity, options, deps = {}) {
|
|
8275
9242
|
const tableLayout = resolveTableLayout(options);
|
|
8276
9243
|
return prepareReport({
|
|
8277
9244
|
commandOptions: options,
|
|
8278
9245
|
supportedFormats: usageReportFormats,
|
|
8279
|
-
buildData: () => buildUsageData(granularity, options),
|
|
9246
|
+
buildData: () => buildUsageData(granularity, options, deps),
|
|
8280
9247
|
getDiagnostics: (usageData) => usageData.diagnostics,
|
|
9248
|
+
runtimeProfile: deps.runtimeProfile,
|
|
8281
9249
|
createShareArtifact: options.share ? (usageData) => ({
|
|
8282
9250
|
fileName: resolveShareFileName(granularity),
|
|
8283
9251
|
svg: renderUsageShareSvg(usageData, granularity),
|
|
@@ -8290,11 +9258,13 @@ async function prepareUsageReport(granularity, options) {
|
|
|
8290
9258
|
});
|
|
8291
9259
|
}
|
|
8292
9260
|
async function runUsageReport(granularity, options) {
|
|
8293
|
-
const
|
|
9261
|
+
const runtimeProfile = createRuntimeProfileCollector();
|
|
9262
|
+
const preparedReport = await prepareUsageReport(granularity, options, { runtimeProfile });
|
|
8294
9263
|
await runPreparedReport({
|
|
8295
9264
|
preparedReport,
|
|
8296
9265
|
emitCommonDiagnostics: emitDiagnostics,
|
|
8297
9266
|
getEnvVarOverrides: (diagnostics) => diagnostics.activeEnvOverrides,
|
|
9267
|
+
getRuntimeProfile: (diagnostics) => diagnostics.runtimeProfile,
|
|
8298
9268
|
warnOnTerminalOverflow: true
|
|
8299
9269
|
});
|
|
8300
9270
|
}
|
|
@@ -8342,7 +9312,7 @@ function registerSharedReportOptions(command, profile) {
|
|
|
8342
9312
|
collectRepeatedOption
|
|
8343
9313
|
).option("--since <YYYY-MM-DD>", "Inclusive start date filter").option("--until <YYYY-MM-DD>", "Inclusive end date filter").option("--timezone <iana>", "Timezone for bucketing", defaultTimezone).option(
|
|
8344
9314
|
"--provider <name>",
|
|
8345
|
-
"Billing-provider filter (
|
|
9315
|
+
"Billing-provider filter (normalized to billing entity; e.g. openai, anthropic, google)"
|
|
8346
9316
|
).option(
|
|
8347
9317
|
"--model <name>",
|
|
8348
9318
|
"Filter by model (repeatable/comma-separated; exact when exact match exists after source/provider/date filters, otherwise substring)",
|
|
@@ -8661,6 +9631,15 @@ var { packageName, packageVersion } = loadPackageMetadataFromRuntime();
|
|
|
8661
9631
|
var updateRuntimeConfig = getUpdateNotifierRuntimeConfig();
|
|
8662
9632
|
var cli = createCli({ version: packageVersion });
|
|
8663
9633
|
async function main() {
|
|
9634
|
+
if (process.env[UPDATE_CHECK_REFRESH_ENV_VAR] === "1") {
|
|
9635
|
+
await refreshUpdateCheckCache({
|
|
9636
|
+
packageName,
|
|
9637
|
+
currentVersion: packageVersion,
|
|
9638
|
+
cacheTtlMs: updateRuntimeConfig.cacheTtlMs,
|
|
9639
|
+
fetchTimeoutMs: updateRuntimeConfig.fetchTimeoutMs
|
|
9640
|
+
});
|
|
9641
|
+
return;
|
|
9642
|
+
}
|
|
8664
9643
|
const updateResult = await checkForUpdatesAndMaybeRestart({
|
|
8665
9644
|
packageName,
|
|
8666
9645
|
currentVersion: packageVersion,
|