@darkhunt-security/endpoint-codex 0.9.13 → 0.9.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex-plugin/plugin.json +1 -1
- package/dist/bin/backfill.mjs +239 -76
- package/dist/bin/enroll.mjs +1 -0
- package/dist/bin/forwarder.mjs +25 -2
- package/dist/bin/guard.mjs +1 -0
- package/dist/bin/init.mjs +1 -0
- package/dist/bin/spool.mjs +1 -0
- package/dist/bin/status.mjs +1 -0
- package/package.json +1 -1
package/dist/bin/backfill.mjs
CHANGED
|
@@ -5674,8 +5674,8 @@ var require_getMachineId_linux = __commonJS({
|
|
|
5674
5674
|
var fs_1 = __require("fs");
|
|
5675
5675
|
var api_1 = (init_esm(), __toCommonJS(esm_exports));
|
|
5676
5676
|
async function getMachineId() {
|
|
5677
|
-
const
|
|
5678
|
-
for (const path of
|
|
5677
|
+
const paths = ["/etc/machine-id", "/var/lib/dbus/machine-id"];
|
|
5678
|
+
for (const path of paths) {
|
|
5679
5679
|
try {
|
|
5680
5680
|
const result2 = await fs_1.promises.readFile(path, { encoding: "utf8" });
|
|
5681
5681
|
return result2.trim();
|
|
@@ -13248,8 +13248,8 @@ var require_getMachineId_linux2 = __commonJS({
|
|
|
13248
13248
|
var fs_1 = __require("fs");
|
|
13249
13249
|
var api_1 = (init_esm(), __toCommonJS(esm_exports));
|
|
13250
13250
|
async function getMachineId() {
|
|
13251
|
-
const
|
|
13252
|
-
for (const path of
|
|
13251
|
+
const paths = ["/etc/machine-id", "/var/lib/dbus/machine-id"];
|
|
13252
|
+
for (const path of paths) {
|
|
13253
13253
|
try {
|
|
13254
13254
|
const result2 = await fs_1.promises.readFile(path, { encoding: "utf8" });
|
|
13255
13255
|
return result2.trim();
|
|
@@ -18096,8 +18096,8 @@ var DEFAULT_PROFILE = "default";
|
|
|
18096
18096
|
var CREDENTIALS_BASENAME = "endpoint-credentials";
|
|
18097
18097
|
var DEFAULT_CREDENTIALS_PATH = join2(CONFIG_DIR, `${CREDENTIALS_BASENAME}.json`);
|
|
18098
18098
|
var CREDENTIALS_ENV = "DARKHUNT_ENDPOINT_CREDENTIALS";
|
|
18099
|
-
function resolveProfile(
|
|
18100
|
-
return
|
|
18099
|
+
function resolveProfile(explicit, fromConfig) {
|
|
18100
|
+
return explicit ?? process.env["DARKHUNT_PROFILE"] ?? fromConfig ?? DEFAULT_PROFILE;
|
|
18101
18101
|
}
|
|
18102
18102
|
function expandPath(path) {
|
|
18103
18103
|
if (path.startsWith("~/"))
|
|
@@ -18109,13 +18109,13 @@ function credentialsSelection(sources = {}) {
|
|
|
18109
18109
|
const pointer = s.explicitPath ?? process.env[CREDENTIALS_ENV] ?? s.configuredPath;
|
|
18110
18110
|
if (pointer)
|
|
18111
18111
|
return { path: expandPath(pointer), explicit: true };
|
|
18112
|
-
const
|
|
18113
|
-
if (
|
|
18112
|
+
const profile = s.profile ?? DEFAULT_PROFILE;
|
|
18113
|
+
if (profile === DEFAULT_PROFILE)
|
|
18114
18114
|
return { path: DEFAULT_CREDENTIALS_PATH, explicit: false };
|
|
18115
|
-
return { path: join2(CONFIG_DIR, `${CREDENTIALS_BASENAME}.${
|
|
18115
|
+
return { path: join2(CONFIG_DIR, `${CREDENTIALS_BASENAME}.${profile}.json`), explicit: true };
|
|
18116
18116
|
}
|
|
18117
|
-
function scopeKey(vendor,
|
|
18118
|
-
return
|
|
18117
|
+
function scopeKey(vendor, profile) {
|
|
18118
|
+
return profile === DEFAULT_PROFILE ? vendor : `${vendor}.${profile}`;
|
|
18119
18119
|
}
|
|
18120
18120
|
|
|
18121
18121
|
// packages/core/dist/config/local.js
|
|
@@ -18175,11 +18175,11 @@ function readConfigFile(vendor) {
|
|
|
18175
18175
|
}
|
|
18176
18176
|
function loadLocalConfig(vendor, options = {}) {
|
|
18177
18177
|
const vendorFile = readConfigFile(vendor);
|
|
18178
|
-
const
|
|
18178
|
+
const profile = resolveProfile(options.profile, vendorFile.profile);
|
|
18179
18179
|
const selection = credentialsSelection({
|
|
18180
18180
|
...options.credentialsPath !== void 0 ? { explicitPath: options.credentialsPath } : {},
|
|
18181
18181
|
...vendorFile.credentials !== void 0 ? { configuredPath: vendorFile.credentials } : {},
|
|
18182
|
-
profile
|
|
18182
|
+
profile
|
|
18183
18183
|
});
|
|
18184
18184
|
const inlineComplete = REQUIRED_FIELDS.every((name) => typeof vendorFile[name] === "string" && vendorFile[name] !== "");
|
|
18185
18185
|
const useInline = inlineComplete && !selection.explicit;
|
|
@@ -18198,13 +18198,13 @@ function loadLocalConfig(vendor, options = {}) {
|
|
|
18198
18198
|
["applicationId", applicationId]
|
|
18199
18199
|
]) {
|
|
18200
18200
|
if (typeof value !== "string" || value === "") {
|
|
18201
|
-
throw new ConfigError(useInline ? `missing ${name} in ${configPath()}` : `missing ${name} for profile '${
|
|
18201
|
+
throw new ConfigError(useInline ? `missing ${name} in ${configPath()}` : `missing ${name} for profile '${profile}' (looked in ${credsPath}, then ${configPath()})`);
|
|
18202
18202
|
}
|
|
18203
18203
|
}
|
|
18204
18204
|
return {
|
|
18205
|
-
profile
|
|
18205
|
+
profile,
|
|
18206
18206
|
credentialsPath: credsPath,
|
|
18207
|
-
scope: scopeKey(vendor,
|
|
18207
|
+
scope: scopeKey(vendor, profile),
|
|
18208
18208
|
baseUrl,
|
|
18209
18209
|
apiKey,
|
|
18210
18210
|
tenantId,
|
|
@@ -18245,16 +18245,16 @@ function loadBeat(scope) {
|
|
|
18245
18245
|
function liveTranscripts(beat, now = Date.now()) {
|
|
18246
18246
|
if (!beat)
|
|
18247
18247
|
return [];
|
|
18248
|
-
const
|
|
18248
|
+
const paths = /* @__PURE__ */ new Set();
|
|
18249
18249
|
for (const session of Object.values(beat.sessions ?? {})) {
|
|
18250
18250
|
if (!session.transcriptPath)
|
|
18251
18251
|
continue;
|
|
18252
18252
|
const at = Date.parse(session.at);
|
|
18253
18253
|
if (Number.isNaN(at) || now - at > BEAT_LIVE_MS)
|
|
18254
18254
|
continue;
|
|
18255
|
-
|
|
18255
|
+
paths.add(session.transcriptPath);
|
|
18256
18256
|
}
|
|
18257
|
-
return [...
|
|
18257
|
+
return [...paths];
|
|
18258
18258
|
}
|
|
18259
18259
|
|
|
18260
18260
|
// packages/core/dist/emit/ids.js
|
|
@@ -18880,6 +18880,19 @@ function isAlive(pid) {
|
|
|
18880
18880
|
return false;
|
|
18881
18881
|
}
|
|
18882
18882
|
}
|
|
18883
|
+
var DEFAULT_LOCK_WAIT_MS = 60 * 1e3;
|
|
18884
|
+
async function acquireLockWithin(scope, timeoutMs, sleep2 = (ms) => new Promise((r) => setTimeout(r, ms)), now = Date.now) {
|
|
18885
|
+
const deadline = now() + timeoutMs;
|
|
18886
|
+
for (; ; ) {
|
|
18887
|
+
const release = acquireLock(scope);
|
|
18888
|
+
if (release)
|
|
18889
|
+
return release;
|
|
18890
|
+
const remaining = deadline - now();
|
|
18891
|
+
if (remaining <= 0)
|
|
18892
|
+
return null;
|
|
18893
|
+
await sleep2(Math.min(250, remaining));
|
|
18894
|
+
}
|
|
18895
|
+
}
|
|
18883
18896
|
|
|
18884
18897
|
// packages/core/dist/forwarder/export-status.js
|
|
18885
18898
|
init_esm();
|
|
@@ -21270,16 +21283,16 @@ var Generation = class extends Span {
|
|
|
21270
21283
|
this.otelSpan.setAttribute(ATTR.MODEL_NAME, model);
|
|
21271
21284
|
this.otelSpan.setAttribute(GEN_AI.REQUEST_MODEL, model);
|
|
21272
21285
|
}
|
|
21273
|
-
setUsage(
|
|
21274
|
-
this.otelSpan.setAttribute(ATTR.USAGE_DETAILS, safeJsonStringify(
|
|
21275
|
-
if (
|
|
21276
|
-
this.otelSpan.setAttribute(GEN_AI.USAGE_INPUT_TOKENS,
|
|
21277
|
-
if (
|
|
21278
|
-
this.otelSpan.setAttribute(GEN_AI.USAGE_OUTPUT_TOKENS,
|
|
21279
|
-
if (
|
|
21280
|
-
this.otelSpan.setAttribute(GEN_AI.USAGE_CACHE_READ_INPUT_TOKENS,
|
|
21281
|
-
if (
|
|
21282
|
-
this.otelSpan.setAttribute(GEN_AI.USAGE_CACHE_CREATION_INPUT_TOKENS,
|
|
21286
|
+
setUsage(usage2) {
|
|
21287
|
+
this.otelSpan.setAttribute(ATTR.USAGE_DETAILS, safeJsonStringify(usage2));
|
|
21288
|
+
if (usage2.input_tokens !== void 0)
|
|
21289
|
+
this.otelSpan.setAttribute(GEN_AI.USAGE_INPUT_TOKENS, usage2.input_tokens);
|
|
21290
|
+
if (usage2.output_tokens !== void 0)
|
|
21291
|
+
this.otelSpan.setAttribute(GEN_AI.USAGE_OUTPUT_TOKENS, usage2.output_tokens);
|
|
21292
|
+
if (usage2.cache_read_tokens !== void 0)
|
|
21293
|
+
this.otelSpan.setAttribute(GEN_AI.USAGE_CACHE_READ_INPUT_TOKENS, usage2.cache_read_tokens);
|
|
21294
|
+
if (usage2.cache_creation_tokens !== void 0)
|
|
21295
|
+
this.otelSpan.setAttribute(GEN_AI.USAGE_CACHE_CREATION_INPUT_TOKENS, usage2.cache_creation_tokens);
|
|
21283
21296
|
}
|
|
21284
21297
|
setCost(cost) {
|
|
21285
21298
|
this.otelSpan.setAttribute(ATTR.COST_DETAILS, safeJsonStringify(cost));
|
|
@@ -21683,19 +21696,19 @@ var SPOOL_KEY = "::spool::";
|
|
|
21683
21696
|
var DARKHUNT_SOURCE_HINT = "darkhunt-telemetry";
|
|
21684
21697
|
function drainSpool(mapper, checkpoints) {
|
|
21685
21698
|
const path = spoolPath(mapper.vendor);
|
|
21686
|
-
const
|
|
21699
|
+
const paths = /* @__PURE__ */ new Set();
|
|
21687
21700
|
try {
|
|
21688
21701
|
const { lines, offset, size } = readNewLines(path, resumeOffset(checkpoints, SPOOL_KEY));
|
|
21689
21702
|
for (const line of lines) {
|
|
21690
21703
|
try {
|
|
21691
21704
|
const event = JSON.parse(line.text);
|
|
21692
21705
|
if (event.transcriptPath)
|
|
21693
|
-
|
|
21706
|
+
paths.add(event.transcriptPath);
|
|
21694
21707
|
} catch {
|
|
21695
21708
|
continue;
|
|
21696
21709
|
}
|
|
21697
21710
|
}
|
|
21698
|
-
return { paths: [...
|
|
21711
|
+
return { paths: [...paths], checkpoint: { offset, size } };
|
|
21699
21712
|
} catch {
|
|
21700
21713
|
return { paths: [] };
|
|
21701
21714
|
}
|
|
@@ -21711,7 +21724,9 @@ function tryLoadConfig(vendor, options) {
|
|
|
21711
21724
|
}
|
|
21712
21725
|
async function runForwarder(mapper, options = {}) {
|
|
21713
21726
|
const config = options.config ?? tryLoadConfig(mapper.vendor, options);
|
|
21714
|
-
const
|
|
21727
|
+
const scope = config?.scope ?? mapper.vendor;
|
|
21728
|
+
const waited = options.lockWaitMs !== void 0 && options.lockWaitMs > 0;
|
|
21729
|
+
const release = waited ? await acquireLockWithin(scope, options.lockWaitMs) : acquireLock(scope);
|
|
21715
21730
|
if (!release) {
|
|
21716
21731
|
return {
|
|
21717
21732
|
transcripts: 0,
|
|
@@ -21719,7 +21734,13 @@ async function runForwarder(mapper, options = {}) {
|
|
|
21719
21734
|
skipped: ["another forwarder holds the lock"],
|
|
21720
21735
|
kinds: {},
|
|
21721
21736
|
warnings: [],
|
|
21722
|
-
|
|
21737
|
+
emitted: { transcripts: 0, records: 0 },
|
|
21738
|
+
lockBusy: true,
|
|
21739
|
+
// A caller that waited and still did not get it has had its request dropped, and
|
|
21740
|
+
// must not exit zero on it. One that never asked to wait was simply beaten to
|
|
21741
|
+
// work that is now someone else's, which is success.
|
|
21742
|
+
ok: !waited,
|
|
21743
|
+
...waited ? { error: "another forwarder holds the lock" } : {}
|
|
21723
21744
|
};
|
|
21724
21745
|
}
|
|
21725
21746
|
const result2 = {
|
|
@@ -21728,6 +21749,7 @@ async function runForwarder(mapper, options = {}) {
|
|
|
21728
21749
|
skipped: [],
|
|
21729
21750
|
kinds: {},
|
|
21730
21751
|
warnings: [],
|
|
21752
|
+
emitted: { transcripts: 0, records: 0 },
|
|
21731
21753
|
ok: true
|
|
21732
21754
|
};
|
|
21733
21755
|
try {
|
|
@@ -21784,8 +21806,8 @@ async function runForwarder(mapper, options = {}) {
|
|
|
21784
21806
|
}
|
|
21785
21807
|
const drain = options.paths ? { paths: options.paths } : drainSpool(mapper, checkpoints);
|
|
21786
21808
|
const named = options.paths ? drain.paths : [.../* @__PURE__ */ new Set([...drain.paths, ...liveTranscripts(loadBeat(config.scope))])];
|
|
21787
|
-
const
|
|
21788
|
-
if (
|
|
21809
|
+
const paths = [...new Set(named.flatMap((path) => [path, ...related(mapper, path)]))];
|
|
21810
|
+
if (paths.length === 0) {
|
|
21789
21811
|
if (drain.checkpoint)
|
|
21790
21812
|
checkpoints[SPOOL_KEY] = drain.checkpoint;
|
|
21791
21813
|
saveCheckpoints(config.scope, checkpoints);
|
|
@@ -21815,7 +21837,7 @@ async function runForwarder(mapper, options = {}) {
|
|
|
21815
21837
|
const watch = watchExports();
|
|
21816
21838
|
const staged = { ...checkpoints };
|
|
21817
21839
|
try {
|
|
21818
|
-
for (const path of
|
|
21840
|
+
for (const path of paths) {
|
|
21819
21841
|
try {
|
|
21820
21842
|
const from = resumeOffset(checkpoints, path);
|
|
21821
21843
|
const carried = resumeTurn(checkpoints, path, from);
|
|
@@ -21877,6 +21899,7 @@ async function runForwarder(mapper, options = {}) {
|
|
|
21877
21899
|
await client.shutdown();
|
|
21878
21900
|
watch.stop();
|
|
21879
21901
|
}
|
|
21902
|
+
result2.emitted = { transcripts: result2.transcripts, records: result2.records };
|
|
21880
21903
|
if (watch.failures() > 0) {
|
|
21881
21904
|
result2.ok = false;
|
|
21882
21905
|
result2.error = watch.reason() ?? "span export failed";
|
|
@@ -21901,12 +21924,12 @@ async function runForwarder(mapper, options = {}) {
|
|
|
21901
21924
|
release();
|
|
21902
21925
|
}
|
|
21903
21926
|
}
|
|
21904
|
-
function withinDays(
|
|
21927
|
+
function withinDays(paths, days) {
|
|
21905
21928
|
if (!Number.isFinite(days) || days <= 0) {
|
|
21906
21929
|
throw new RangeError(`--days must be a positive number of days, got ${days}`);
|
|
21907
21930
|
}
|
|
21908
21931
|
const cutoff = Date.now() - days * 24 * 60 * 60 * 1e3;
|
|
21909
|
-
return
|
|
21932
|
+
return paths.filter((path) => {
|
|
21910
21933
|
try {
|
|
21911
21934
|
return statSync4(path).mtimeMs >= cutoff;
|
|
21912
21935
|
} catch {
|
|
@@ -21958,6 +21981,148 @@ import { homedir as homedir3 } from "node:os";
|
|
|
21958
21981
|
import { join as join9 } from "node:path";
|
|
21959
21982
|
var CLI_CREDENTIALS_PATH = join9(homedir3(), ".darkhunt", "credentials.json");
|
|
21960
21983
|
|
|
21984
|
+
// packages/core/dist/cli/backfill.js
|
|
21985
|
+
var BOOLEAN_FLAGS = /* @__PURE__ */ new Set(["dry-run"]);
|
|
21986
|
+
var VALUE_FLAGS = /* @__PURE__ */ new Set(["profile", "credentials", "days", "batch"]);
|
|
21987
|
+
var DEFAULT_BATCH = 25;
|
|
21988
|
+
function usage(command) {
|
|
21989
|
+
return [
|
|
21990
|
+
`usage: ${command} [options] [paths...]`,
|
|
21991
|
+
"",
|
|
21992
|
+
" --dry-run map only: no network, no checkpoints",
|
|
21993
|
+
" --days=N only transcripts written to in the last N days",
|
|
21994
|
+
" --batch=N transcripts per pass (default " + String(DEFAULT_BATCH) + ")",
|
|
21995
|
+
" --profile=NAME endpoint profile to ship to",
|
|
21996
|
+
" --credentials=PATH explicit credentials file",
|
|
21997
|
+
" -h, --help this message",
|
|
21998
|
+
"",
|
|
21999
|
+
" With no paths this ships EVERY transcript on disk. Narrow it with --days",
|
|
22000
|
+
" or by naming transcripts, and confirm the destination with --dry-run first."
|
|
22001
|
+
].join("\n");
|
|
22002
|
+
}
|
|
22003
|
+
function parseBackfillArgs(argv) {
|
|
22004
|
+
const paths = [];
|
|
22005
|
+
const values = /* @__PURE__ */ new Map();
|
|
22006
|
+
let dryRun = false;
|
|
22007
|
+
for (const arg of argv) {
|
|
22008
|
+
if (arg === "-h" || arg === "--help")
|
|
22009
|
+
return { kind: "help" };
|
|
22010
|
+
if (!arg.startsWith("--")) {
|
|
22011
|
+
paths.push(arg);
|
|
22012
|
+
continue;
|
|
22013
|
+
}
|
|
22014
|
+
const body = arg.slice(2);
|
|
22015
|
+
const eq = body.indexOf("=");
|
|
22016
|
+
const name = eq === -1 ? body : body.slice(0, eq);
|
|
22017
|
+
if (BOOLEAN_FLAGS.has(name)) {
|
|
22018
|
+
if (eq !== -1)
|
|
22019
|
+
return { kind: "error", message: `--${name} takes no value` };
|
|
22020
|
+
dryRun = true;
|
|
22021
|
+
continue;
|
|
22022
|
+
}
|
|
22023
|
+
if (VALUE_FLAGS.has(name)) {
|
|
22024
|
+
if (eq === -1)
|
|
22025
|
+
return { kind: "error", message: `--${name} needs a value, as --${name}=VALUE` };
|
|
22026
|
+
values.set(name, body.slice(eq + 1));
|
|
22027
|
+
continue;
|
|
22028
|
+
}
|
|
22029
|
+
return {
|
|
22030
|
+
kind: "error",
|
|
22031
|
+
message: `unknown option '${arg}'. Known: ${[...BOOLEAN_FLAGS, ...VALUE_FLAGS].map((f) => `--${f}`).sort().join(", ")}, --help`
|
|
22032
|
+
};
|
|
22033
|
+
}
|
|
22034
|
+
const positive = (name) => {
|
|
22035
|
+
const raw = values.get(name);
|
|
22036
|
+
if (raw === void 0)
|
|
22037
|
+
return void 0;
|
|
22038
|
+
const n = Number(raw);
|
|
22039
|
+
if (!Number.isFinite(n) || n <= 0) {
|
|
22040
|
+
return { kind: "error", message: `--${name} must be a positive number, got '${raw}'` };
|
|
22041
|
+
}
|
|
22042
|
+
return n;
|
|
22043
|
+
};
|
|
22044
|
+
const sinceDays = positive("days");
|
|
22045
|
+
if (typeof sinceDays === "object")
|
|
22046
|
+
return sinceDays;
|
|
22047
|
+
const batch = positive("batch");
|
|
22048
|
+
if (typeof batch === "object")
|
|
22049
|
+
return batch;
|
|
22050
|
+
return {
|
|
22051
|
+
kind: "run",
|
|
22052
|
+
options: {
|
|
22053
|
+
dryRun,
|
|
22054
|
+
paths,
|
|
22055
|
+
...values.get("profile") !== void 0 ? { profile: values.get("profile") } : {},
|
|
22056
|
+
...values.get("credentials") !== void 0 ? { credentialsPath: values.get("credentials") } : {},
|
|
22057
|
+
...sinceDays !== void 0 ? { sinceDays } : {},
|
|
22058
|
+
...batch !== void 0 ? { batch } : {}
|
|
22059
|
+
}
|
|
22060
|
+
};
|
|
22061
|
+
}
|
|
22062
|
+
function batches(items, size) {
|
|
22063
|
+
const out = [];
|
|
22064
|
+
for (let i = 0; i < items.length; i += size)
|
|
22065
|
+
out.push(items.slice(i, i + size));
|
|
22066
|
+
return out;
|
|
22067
|
+
}
|
|
22068
|
+
async function runBackfill(mapper, options, say2) {
|
|
22069
|
+
const discovered = options.paths.length > 0 ? options.paths : discoverTranscripts(mapper);
|
|
22070
|
+
const paths = options.sinceDays === void 0 ? discovered : withinDays(discovered, options.sinceDays);
|
|
22071
|
+
const window2 = options.sinceDays === void 0 ? "" : ` written to in the last ${options.sinceDays} day(s)`;
|
|
22072
|
+
say2(`${options.dryRun ? "DRY RUN \u2014 nothing will be sent. " : ""}${paths.length} transcript(s)${window2}`);
|
|
22073
|
+
const pass = (subset) => runForwarder(mapper, {
|
|
22074
|
+
paths: subset,
|
|
22075
|
+
dryRun: options.dryRun,
|
|
22076
|
+
// A backfill is an instruction, not an opportunistic spawn: it waits for the lock
|
|
22077
|
+
// that a hook-fired forwarder may hold, and treats never getting it as a failure
|
|
22078
|
+
// rather than reporting the `shipped 0` an up-to-date machine also prints.
|
|
22079
|
+
...options.dryRun ? {} : { lockWaitMs: options.lockWaitMs ?? DEFAULT_LOCK_WAIT_MS },
|
|
22080
|
+
...options.profile !== void 0 ? { profile: options.profile } : {},
|
|
22081
|
+
...options.credentialsPath !== void 0 ? { credentialsPath: options.credentialsPath } : {}
|
|
22082
|
+
});
|
|
22083
|
+
if (options.dryRun)
|
|
22084
|
+
return pass(paths);
|
|
22085
|
+
if (paths.length === 0)
|
|
22086
|
+
return pass([]);
|
|
22087
|
+
return runPasses(batches(paths, options.batch ?? DEFAULT_BATCH), pass, say2);
|
|
22088
|
+
}
|
|
22089
|
+
async function runPasses(groups, pass, say2) {
|
|
22090
|
+
const total = {
|
|
22091
|
+
transcripts: 0,
|
|
22092
|
+
records: 0,
|
|
22093
|
+
skipped: [],
|
|
22094
|
+
kinds: {},
|
|
22095
|
+
warnings: [],
|
|
22096
|
+
emitted: { transcripts: 0, records: 0 },
|
|
22097
|
+
ok: true
|
|
22098
|
+
};
|
|
22099
|
+
for (const [index, group] of groups.entries()) {
|
|
22100
|
+
const result2 = await pass(group);
|
|
22101
|
+
total.transcripts += result2.transcripts;
|
|
22102
|
+
total.records += result2.records;
|
|
22103
|
+
total.emitted.transcripts += result2.emitted.transcripts;
|
|
22104
|
+
total.emitted.records += result2.emitted.records;
|
|
22105
|
+
total.skipped.push(...result2.skipped);
|
|
22106
|
+
total.warnings.push(...result2.warnings);
|
|
22107
|
+
for (const [kind, n] of Object.entries(result2.kinds)) {
|
|
22108
|
+
total.kinds[kind] = (total.kinds[kind] ?? 0) + n;
|
|
22109
|
+
}
|
|
22110
|
+
if (result2.ok === false) {
|
|
22111
|
+
total.ok = false;
|
|
22112
|
+
if (result2.error !== void 0)
|
|
22113
|
+
total.error = result2.error;
|
|
22114
|
+
if (result2.lockBusy === true)
|
|
22115
|
+
total.lockBusy = true;
|
|
22116
|
+
say2(result2.lockBusy === true ? `batch ${index + 1} of ${groups.length} could not start: another forwarder holds the lock` : `stopped in batch ${index + 1} of ${groups.length}; earlier batches are checkpointed`);
|
|
22117
|
+
break;
|
|
22118
|
+
}
|
|
22119
|
+
if (groups.length > 1) {
|
|
22120
|
+
say2(`batch ${index + 1}/${groups.length}: ${result2.records} record(s)`);
|
|
22121
|
+
}
|
|
22122
|
+
}
|
|
22123
|
+
return total;
|
|
22124
|
+
}
|
|
22125
|
+
|
|
21961
22126
|
// adapters/codex/dist/transcript.js
|
|
21962
22127
|
import { readFileSync as readFileSync6 } from "node:fs";
|
|
21963
22128
|
import { basename, join as join10 } from "node:path";
|
|
@@ -22018,15 +22183,15 @@ function budgeted(steps) {
|
|
|
22018
22183
|
return kept;
|
|
22019
22184
|
}
|
|
22020
22185
|
function execEnvelope(text) {
|
|
22021
|
-
let
|
|
22186
|
+
let parsed2;
|
|
22022
22187
|
try {
|
|
22023
|
-
|
|
22188
|
+
parsed2 = JSON.parse(text);
|
|
22024
22189
|
} catch {
|
|
22025
22190
|
return void 0;
|
|
22026
22191
|
}
|
|
22027
|
-
if (!
|
|
22192
|
+
if (!parsed2 || typeof parsed2 !== "object" || Array.isArray(parsed2))
|
|
22028
22193
|
return void 0;
|
|
22029
|
-
const bag =
|
|
22194
|
+
const bag = parsed2;
|
|
22030
22195
|
if (!Object.prototype.hasOwnProperty.call(bag, "output"))
|
|
22031
22196
|
return void 0;
|
|
22032
22197
|
const metadata = bag["metadata"];
|
|
@@ -22065,13 +22230,13 @@ function jwtClaims(token) {
|
|
|
22065
22230
|
}
|
|
22066
22231
|
}
|
|
22067
22232
|
function emailFromAuth(raw) {
|
|
22068
|
-
let
|
|
22233
|
+
let parsed2;
|
|
22069
22234
|
try {
|
|
22070
|
-
|
|
22235
|
+
parsed2 = JSON.parse(raw);
|
|
22071
22236
|
} catch {
|
|
22072
22237
|
return void 0;
|
|
22073
22238
|
}
|
|
22074
|
-
const tokens =
|
|
22239
|
+
const tokens = parsed2?.tokens;
|
|
22075
22240
|
const token = tokens?.id_token;
|
|
22076
22241
|
if (typeof token !== "string" || token === "")
|
|
22077
22242
|
return void 0;
|
|
@@ -22089,11 +22254,11 @@ function repositoryIdentity(url) {
|
|
|
22089
22254
|
return `${scp[1]}/${stripSuffix(scp[2])}`;
|
|
22090
22255
|
const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `ssh://${trimmed}`;
|
|
22091
22256
|
try {
|
|
22092
|
-
const
|
|
22093
|
-
if (!
|
|
22257
|
+
const parsed2 = new URL(withScheme);
|
|
22258
|
+
if (!parsed2.hostname)
|
|
22094
22259
|
return trimmed;
|
|
22095
|
-
const path = stripSuffix(
|
|
22096
|
-
return path === "" ?
|
|
22260
|
+
const path = stripSuffix(parsed2.pathname.replace(/^\/+/, ""));
|
|
22261
|
+
return path === "" ? parsed2.hostname : `${parsed2.hostname}/${path}`;
|
|
22097
22262
|
} catch {
|
|
22098
22263
|
return trimmed;
|
|
22099
22264
|
}
|
|
@@ -22201,8 +22366,8 @@ var codexTranscript = {
|
|
|
22201
22366
|
}
|
|
22202
22367
|
const kind = str(payload["type"]);
|
|
22203
22368
|
if (raw.type === "event_msg" && kind === "token_count") {
|
|
22204
|
-
const
|
|
22205
|
-
return
|
|
22369
|
+
const usage2 = mapTokenCount(payload["info"]);
|
|
22370
|
+
return usage2 ? [{ ...base, kind: "usage", usage: usage2 }] : [];
|
|
22206
22371
|
}
|
|
22207
22372
|
if (raw.type === "compacted") {
|
|
22208
22373
|
const note = str(payload["message"]);
|
|
@@ -22337,33 +22502,21 @@ var codexTranscript = {
|
|
|
22337
22502
|
};
|
|
22338
22503
|
|
|
22339
22504
|
// adapters/codex/bin/backfill.mjs
|
|
22340
|
-
var argv = process.argv.slice(2);
|
|
22341
|
-
var dryRun = argv.includes("--dry-run");
|
|
22342
|
-
var explicit = argv.filter((a) => !a.startsWith("--"));
|
|
22343
|
-
var flag = (n) => {
|
|
22344
|
-
const a = argv.find((x) => x.startsWith(`--${n}=`));
|
|
22345
|
-
return a ? a.slice(n.length + 3) : void 0;
|
|
22346
|
-
};
|
|
22347
|
-
var profile = flag("profile");
|
|
22348
|
-
var credentialsPath2 = flag("credentials");
|
|
22349
|
-
var sinceDays;
|
|
22350
|
-
if (flag("days") !== void 0) {
|
|
22351
|
-
sinceDays = Number(flag("days"));
|
|
22352
|
-
if (!Number.isFinite(sinceDays) || sinceDays <= 0) {
|
|
22353
|
-
process.stderr.write(`darkhunt: --days must be a positive number, got '${flag("days")}'
|
|
22354
|
-
`);
|
|
22355
|
-
process.exit(1);
|
|
22356
|
-
}
|
|
22357
|
-
}
|
|
22358
|
-
var discovered = explicit.length > 0 ? explicit : discoverTranscripts(codexTranscript);
|
|
22359
|
-
var paths = sinceDays === void 0 ? discovered : withinDays(discovered, sinceDays);
|
|
22360
|
-
var window2 = sinceDays === void 0 ? "" : ` written to in the last ${sinceDays} day(s)`;
|
|
22361
22505
|
var say = (m) => process.stderr.write(`darkhunt: ${m}
|
|
22362
22506
|
`);
|
|
22363
|
-
|
|
22364
|
-
|
|
22507
|
+
var parsed = parseBackfillArgs(process.argv.slice(2));
|
|
22508
|
+
if (parsed.kind === "help") {
|
|
22509
|
+
process.stdout.write(usage("backfill.mjs") + "\n");
|
|
22510
|
+
process.exit(0);
|
|
22511
|
+
}
|
|
22512
|
+
if (parsed.kind === "error") {
|
|
22513
|
+
say(parsed.message);
|
|
22514
|
+
process.stderr.write(usage("backfill.mjs") + "\n");
|
|
22515
|
+
process.exit(2);
|
|
22516
|
+
}
|
|
22517
|
+
var result = await runBackfill(codexTranscript, parsed.options, say);
|
|
22365
22518
|
say(
|
|
22366
|
-
`${dryRun ? "mapped" : "shipped"} ${result.records} record(s) from ${result.transcripts} transcript(s)`
|
|
22519
|
+
`${parsed.options.dryRun ? "mapped" : "shipped"} ${result.records} record(s) from ${result.transcripts} transcript(s)`
|
|
22367
22520
|
);
|
|
22368
22521
|
for (const [kind, n] of Object.entries(result.kinds).sort((a, b) => b[1] - a[1])) {
|
|
22369
22522
|
say(` ${kind.padEnd(18)} ${n}`);
|
|
@@ -22371,7 +22524,17 @@ for (const [kind, n] of Object.entries(result.kinds).sort((a, b) => b[1] - a[1])
|
|
|
22371
22524
|
for (const skip of result.skipped) say(`skipped ${skip}`);
|
|
22372
22525
|
for (const warning of result.warnings ?? []) say(`WARNING ${warning}`);
|
|
22373
22526
|
if (result.ok === false) {
|
|
22527
|
+
if (result.lockBusy === true) {
|
|
22528
|
+
say("NOT RUN \u2014 another forwarder held the lock for the whole wait");
|
|
22529
|
+
say("nothing was shipped; re-run when the capture lane is idle");
|
|
22530
|
+
process.exit(3);
|
|
22531
|
+
}
|
|
22374
22532
|
say(`EXPORT FAILED \u2014 ${result.error}`);
|
|
22533
|
+
if (result.emitted.records > 0) {
|
|
22534
|
+
say(
|
|
22535
|
+
`${result.emitted.records} record(s) from ${result.emitted.transcripts} transcript(s) were sent before the failure and may have been stored`
|
|
22536
|
+
);
|
|
22537
|
+
}
|
|
22375
22538
|
say("checkpoints were NOT advanced; re-run once the cause is fixed");
|
|
22376
22539
|
process.exit(1);
|
|
22377
22540
|
}
|
package/dist/bin/enroll.mjs
CHANGED
package/dist/bin/forwarder.mjs
CHANGED
|
@@ -18880,6 +18880,19 @@ function isAlive(pid) {
|
|
|
18880
18880
|
return false;
|
|
18881
18881
|
}
|
|
18882
18882
|
}
|
|
18883
|
+
var DEFAULT_LOCK_WAIT_MS = 60 * 1e3;
|
|
18884
|
+
async function acquireLockWithin(scope, timeoutMs, sleep2 = (ms) => new Promise((r) => setTimeout(r, ms)), now = Date.now) {
|
|
18885
|
+
const deadline = now() + timeoutMs;
|
|
18886
|
+
for (; ; ) {
|
|
18887
|
+
const release = acquireLock(scope);
|
|
18888
|
+
if (release)
|
|
18889
|
+
return release;
|
|
18890
|
+
const remaining = deadline - now();
|
|
18891
|
+
if (remaining <= 0)
|
|
18892
|
+
return null;
|
|
18893
|
+
await sleep2(Math.min(250, remaining));
|
|
18894
|
+
}
|
|
18895
|
+
}
|
|
18883
18896
|
|
|
18884
18897
|
// packages/core/dist/forwarder/export-status.js
|
|
18885
18898
|
init_esm();
|
|
@@ -21711,7 +21724,9 @@ function tryLoadConfig(vendor, options) {
|
|
|
21711
21724
|
}
|
|
21712
21725
|
async function runForwarder(mapper, options = {}) {
|
|
21713
21726
|
const config = options.config ?? tryLoadConfig(mapper.vendor, options);
|
|
21714
|
-
const
|
|
21727
|
+
const scope = config?.scope ?? mapper.vendor;
|
|
21728
|
+
const waited = options.lockWaitMs !== void 0 && options.lockWaitMs > 0;
|
|
21729
|
+
const release = waited ? await acquireLockWithin(scope, options.lockWaitMs) : acquireLock(scope);
|
|
21715
21730
|
if (!release) {
|
|
21716
21731
|
return {
|
|
21717
21732
|
transcripts: 0,
|
|
@@ -21719,7 +21734,13 @@ async function runForwarder(mapper, options = {}) {
|
|
|
21719
21734
|
skipped: ["another forwarder holds the lock"],
|
|
21720
21735
|
kinds: {},
|
|
21721
21736
|
warnings: [],
|
|
21722
|
-
|
|
21737
|
+
emitted: { transcripts: 0, records: 0 },
|
|
21738
|
+
lockBusy: true,
|
|
21739
|
+
// A caller that waited and still did not get it has had its request dropped, and
|
|
21740
|
+
// must not exit zero on it. One that never asked to wait was simply beaten to
|
|
21741
|
+
// work that is now someone else's, which is success.
|
|
21742
|
+
ok: !waited,
|
|
21743
|
+
...waited ? { error: "another forwarder holds the lock" } : {}
|
|
21723
21744
|
};
|
|
21724
21745
|
}
|
|
21725
21746
|
const result = {
|
|
@@ -21728,6 +21749,7 @@ async function runForwarder(mapper, options = {}) {
|
|
|
21728
21749
|
skipped: [],
|
|
21729
21750
|
kinds: {},
|
|
21730
21751
|
warnings: [],
|
|
21752
|
+
emitted: { transcripts: 0, records: 0 },
|
|
21731
21753
|
ok: true
|
|
21732
21754
|
};
|
|
21733
21755
|
try {
|
|
@@ -21877,6 +21899,7 @@ async function runForwarder(mapper, options = {}) {
|
|
|
21877
21899
|
await client.shutdown();
|
|
21878
21900
|
watch.stop();
|
|
21879
21901
|
}
|
|
21902
|
+
result.emitted = { transcripts: result.transcripts, records: result.records };
|
|
21880
21903
|
if (watch.failures() > 0) {
|
|
21881
21904
|
result.ok = false;
|
|
21882
21905
|
result.error = watch.reason() ?? "span export failed";
|
package/dist/bin/guard.mjs
CHANGED
package/dist/bin/init.mjs
CHANGED
package/dist/bin/spool.mjs
CHANGED
package/dist/bin/status.mjs
CHANGED
|
@@ -190,6 +190,7 @@ var LINE_CEILING = 64 * 1024 * 1024;
|
|
|
190
190
|
|
|
191
191
|
// packages/core/dist/forwarder/lock.js
|
|
192
192
|
var STALE_MS = 5 * 60 * 1e3;
|
|
193
|
+
var DEFAULT_LOCK_WAIT_MS = 60 * 1e3;
|
|
193
194
|
|
|
194
195
|
// packages/core/dist/forwarder/health.js
|
|
195
196
|
import { mkdirSync as mkdirSync3, readFileSync as readFileSync4, renameSync as renameSync3, writeFileSync as writeFileSync3 } from "node:fs";
|
package/package.json
CHANGED