modelstat 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +338 -67
- package/dist/cli.mjs.map +1 -1
- package/package.json +3 -3
package/dist/cli.mjs
CHANGED
|
@@ -125,8 +125,12 @@ async function resolveGitContext(cwd) {
|
|
|
125
125
|
function guessRepoSlugFromPath(cwd) {
|
|
126
126
|
if (!cwd) return null;
|
|
127
127
|
const m = /\/(?:www|src|code|repos|projects)\/([^/]+)\/([^/]+)/i.exec(cwd);
|
|
128
|
-
if (m) return
|
|
129
|
-
|
|
128
|
+
if (!m) return null;
|
|
129
|
+
const a = m[1];
|
|
130
|
+
const b = m[2];
|
|
131
|
+
if (!a || !b) return null;
|
|
132
|
+
if (b.startsWith(".") || b === "worktrees") return a;
|
|
133
|
+
return `${a}/${b}`;
|
|
130
134
|
}
|
|
131
135
|
var pexec, cache;
|
|
132
136
|
var init_git = __esm({
|
|
@@ -16730,7 +16734,7 @@ var require_client_h2 = __commonJS({
|
|
|
16730
16734
|
"../../node_modules/.pnpm/undici@7.25.0/node_modules/undici/lib/dispatcher/client-h2.js"(exports, module) {
|
|
16731
16735
|
"use strict";
|
|
16732
16736
|
var assert = __require("assert");
|
|
16733
|
-
var { pipeline } = __require("stream");
|
|
16737
|
+
var { pipeline: pipeline2 } = __require("stream");
|
|
16734
16738
|
var util2 = require_util();
|
|
16735
16739
|
var {
|
|
16736
16740
|
RequestContentLengthMismatchError,
|
|
@@ -17363,7 +17367,7 @@ var require_client_h2 = __commonJS({
|
|
|
17363
17367
|
}
|
|
17364
17368
|
function writeStream(abort, socket, expectsPayload, h2stream, body, client, request3, contentLength) {
|
|
17365
17369
|
assert(contentLength !== 0 || client[kRunning] === 0, "stream body cannot be pipelined");
|
|
17366
|
-
const pipe =
|
|
17370
|
+
const pipe = pipeline2(
|
|
17367
17371
|
body,
|
|
17368
17372
|
h2stream,
|
|
17369
17373
|
(err) => {
|
|
@@ -18070,10 +18074,10 @@ var require_pool_base = __commonJS({
|
|
|
18070
18074
|
[kClients] = [];
|
|
18071
18075
|
[kNeedDrain] = false;
|
|
18072
18076
|
[kOnDrain](client, origin, targets) {
|
|
18073
|
-
const
|
|
18077
|
+
const queue2 = this[kQueue];
|
|
18074
18078
|
let needDrain = false;
|
|
18075
18079
|
while (!needDrain) {
|
|
18076
|
-
const item =
|
|
18080
|
+
const item = queue2.shift();
|
|
18077
18081
|
if (!item) {
|
|
18078
18082
|
break;
|
|
18079
18083
|
}
|
|
@@ -18085,7 +18089,7 @@ var require_pool_base = __commonJS({
|
|
|
18085
18089
|
this[kNeedDrain] = false;
|
|
18086
18090
|
this.emit("drain", origin, [this, ...targets]);
|
|
18087
18091
|
}
|
|
18088
|
-
if (this[kClosedResolve] &&
|
|
18092
|
+
if (this[kClosedResolve] && queue2.isEmpty()) {
|
|
18089
18093
|
const closeAll = [];
|
|
18090
18094
|
for (let i = 0; i < this[kClients].length; i++) {
|
|
18091
18095
|
const client2 = this[kClients][i];
|
|
@@ -21129,7 +21133,7 @@ var require_api_pipeline = __commonJS({
|
|
|
21129
21133
|
util2.destroy(ret, err);
|
|
21130
21134
|
}
|
|
21131
21135
|
};
|
|
21132
|
-
function
|
|
21136
|
+
function pipeline2(opts, handler) {
|
|
21133
21137
|
try {
|
|
21134
21138
|
const pipelineHandler = new PipelineHandler(opts, handler);
|
|
21135
21139
|
this.dispatch({ ...opts, body: pipelineHandler.req }, pipelineHandler);
|
|
@@ -21138,7 +21142,7 @@ var require_api_pipeline = __commonJS({
|
|
|
21138
21142
|
return new PassThrough().destroy(err);
|
|
21139
21143
|
}
|
|
21140
21144
|
}
|
|
21141
|
-
module.exports =
|
|
21145
|
+
module.exports = pipeline2;
|
|
21142
21146
|
}
|
|
21143
21147
|
});
|
|
21144
21148
|
|
|
@@ -22010,13 +22014,13 @@ var require_mock_call_history = __commonJS({
|
|
|
22010
22014
|
"use strict";
|
|
22011
22015
|
var { kMockCallHistoryAddLog } = require_mock_symbols();
|
|
22012
22016
|
var { InvalidArgumentError } = require_errors();
|
|
22013
|
-
function handleFilterCallsWithOptions(criteria, options, handler,
|
|
22017
|
+
function handleFilterCallsWithOptions(criteria, options, handler, store2) {
|
|
22014
22018
|
switch (options.operator) {
|
|
22015
22019
|
case "OR":
|
|
22016
|
-
|
|
22017
|
-
return
|
|
22020
|
+
store2.push(...handler(criteria));
|
|
22021
|
+
return store2;
|
|
22018
22022
|
case "AND":
|
|
22019
|
-
return handler.call({ logs:
|
|
22023
|
+
return handler.call({ logs: store2 }, criteria);
|
|
22020
22024
|
default:
|
|
22021
22025
|
throw new InvalidArgumentError("options.operator must to be a case insensitive string equal to 'OR' or 'AND'");
|
|
22022
22026
|
}
|
|
@@ -24373,12 +24377,12 @@ var require_cache = __commonJS({
|
|
|
24373
24377
|
}
|
|
24374
24378
|
return false;
|
|
24375
24379
|
}
|
|
24376
|
-
function assertCacheStore(
|
|
24377
|
-
if (typeof
|
|
24378
|
-
throw new TypeError(`expected type of ${name} to be a CacheStore, got ${
|
|
24380
|
+
function assertCacheStore(store2, name = "CacheStore") {
|
|
24381
|
+
if (typeof store2 !== "object" || store2 === null) {
|
|
24382
|
+
throw new TypeError(`expected type of ${name} to be a CacheStore, got ${store2 === null ? "null" : typeof store2}`);
|
|
24379
24383
|
}
|
|
24380
24384
|
for (const fn of ["get", "createWriteStream", "delete"]) {
|
|
24381
|
-
if (typeof
|
|
24385
|
+
if (typeof store2[fn] !== "function") {
|
|
24382
24386
|
throw new TypeError(`${name} needs to have a \`${fn}()\` function`);
|
|
24383
24387
|
}
|
|
24384
24388
|
}
|
|
@@ -24982,8 +24986,8 @@ var require_cache_handler = __commonJS({
|
|
|
24982
24986
|
* @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} cacheKey
|
|
24983
24987
|
* @param {import('../../types/dispatcher.d.ts').default.DispatchHandler} handler
|
|
24984
24988
|
*/
|
|
24985
|
-
constructor({ store, type, cacheByDefault }, cacheKey, handler) {
|
|
24986
|
-
this.#store =
|
|
24989
|
+
constructor({ store: store2, type, cacheByDefault }, cacheKey, handler) {
|
|
24990
|
+
this.#store = store2;
|
|
24987
24991
|
this.#cacheType = type;
|
|
24988
24992
|
this.#cacheByDefault = cacheByDefault;
|
|
24989
24993
|
this.#cacheKey = cacheKey;
|
|
@@ -25392,7 +25396,7 @@ var require_memory_cache_store = __commonJS({
|
|
|
25392
25396
|
assertCacheKey(key);
|
|
25393
25397
|
assertCacheValue(val);
|
|
25394
25398
|
const topLevelKey = `${key.origin}:${key.path}`;
|
|
25395
|
-
const
|
|
25399
|
+
const store2 = this;
|
|
25396
25400
|
const entry = { ...key, ...val, body: [], size: 0 };
|
|
25397
25401
|
return new Writable({
|
|
25398
25402
|
write(chunk, encoding, callback) {
|
|
@@ -25400,7 +25404,7 @@ var require_memory_cache_store = __commonJS({
|
|
|
25400
25404
|
chunk = Buffer.from(chunk, encoding);
|
|
25401
25405
|
}
|
|
25402
25406
|
entry.size += chunk.byteLength;
|
|
25403
|
-
if (entry.size >=
|
|
25407
|
+
if (entry.size >= store2.#maxEntrySize) {
|
|
25404
25408
|
this.destroy();
|
|
25405
25409
|
} else {
|
|
25406
25410
|
entry.body.push(chunk);
|
|
@@ -25408,42 +25412,42 @@ var require_memory_cache_store = __commonJS({
|
|
|
25408
25412
|
callback(null);
|
|
25409
25413
|
},
|
|
25410
25414
|
final(callback) {
|
|
25411
|
-
let entries =
|
|
25415
|
+
let entries = store2.#entries.get(topLevelKey);
|
|
25412
25416
|
if (!entries) {
|
|
25413
25417
|
entries = [];
|
|
25414
|
-
|
|
25418
|
+
store2.#entries.set(topLevelKey, entries);
|
|
25415
25419
|
}
|
|
25416
25420
|
const previousEntry = findEntry(key, entries, Date.now());
|
|
25417
25421
|
if (previousEntry) {
|
|
25418
25422
|
const index = entries.indexOf(previousEntry);
|
|
25419
25423
|
entries.splice(index, 1, entry);
|
|
25420
|
-
|
|
25424
|
+
store2.#size -= previousEntry.size;
|
|
25421
25425
|
} else {
|
|
25422
25426
|
entries.push(entry);
|
|
25423
|
-
|
|
25424
|
-
}
|
|
25425
|
-
|
|
25426
|
-
if (
|
|
25427
|
-
if (!
|
|
25428
|
-
|
|
25429
|
-
size:
|
|
25430
|
-
maxSize:
|
|
25431
|
-
count:
|
|
25432
|
-
maxCount:
|
|
25427
|
+
store2.#count += 1;
|
|
25428
|
+
}
|
|
25429
|
+
store2.#size += entry.size;
|
|
25430
|
+
if (store2.#size > store2.#maxSize || store2.#count > store2.#maxCount) {
|
|
25431
|
+
if (!store2.#hasEmittedMaxSizeEvent) {
|
|
25432
|
+
store2.emit("maxSizeExceeded", {
|
|
25433
|
+
size: store2.#size,
|
|
25434
|
+
maxSize: store2.#maxSize,
|
|
25435
|
+
count: store2.#count,
|
|
25436
|
+
maxCount: store2.#maxCount
|
|
25433
25437
|
});
|
|
25434
|
-
|
|
25438
|
+
store2.#hasEmittedMaxSizeEvent = true;
|
|
25435
25439
|
}
|
|
25436
|
-
for (const [key2, entries2] of
|
|
25440
|
+
for (const [key2, entries2] of store2.#entries) {
|
|
25437
25441
|
for (const entry2 of entries2.splice(0, entries2.length / 2)) {
|
|
25438
|
-
|
|
25439
|
-
|
|
25442
|
+
store2.#size -= entry2.size;
|
|
25443
|
+
store2.#count -= 1;
|
|
25440
25444
|
}
|
|
25441
25445
|
if (entries2.length === 0) {
|
|
25442
|
-
|
|
25446
|
+
store2.#entries.delete(key2);
|
|
25443
25447
|
}
|
|
25444
25448
|
}
|
|
25445
|
-
if (
|
|
25446
|
-
|
|
25449
|
+
if (store2.#size < store2.#maxSize && store2.#count < store2.#maxCount) {
|
|
25450
|
+
store2.#hasEmittedMaxSizeEvent = false;
|
|
25447
25451
|
}
|
|
25448
25452
|
}
|
|
25449
25453
|
callback(null);
|
|
@@ -25814,7 +25818,7 @@ var require_cache2 = __commonJS({
|
|
|
25814
25818
|
}
|
|
25815
25819
|
module.exports = (opts = {}) => {
|
|
25816
25820
|
const {
|
|
25817
|
-
store = new MemoryCacheStore(),
|
|
25821
|
+
store: store2 = new MemoryCacheStore(),
|
|
25818
25822
|
methods = ["GET"],
|
|
25819
25823
|
cacheByDefault = void 0,
|
|
25820
25824
|
type = "shared",
|
|
@@ -25823,7 +25827,7 @@ var require_cache2 = __commonJS({
|
|
|
25823
25827
|
if (typeof opts !== "object" || opts === null) {
|
|
25824
25828
|
throw new TypeError(`expected type of opts to be an Object, got ${opts === null ? "null" : typeof opts}`);
|
|
25825
25829
|
}
|
|
25826
|
-
assertCacheStore(
|
|
25830
|
+
assertCacheStore(store2, "opts.store");
|
|
25827
25831
|
assertCacheMethods(methods, "opts.methods");
|
|
25828
25832
|
assertCacheOrigins(origins, "opts.origins");
|
|
25829
25833
|
if (typeof cacheByDefault !== "undefined" && typeof cacheByDefault !== "number") {
|
|
@@ -25833,7 +25837,7 @@ var require_cache2 = __commonJS({
|
|
|
25833
25837
|
throw new TypeError(`expected opts.type to be shared, private, or undefined, got ${typeof type}`);
|
|
25834
25838
|
}
|
|
25835
25839
|
const globalOpts = {
|
|
25836
|
-
store,
|
|
25840
|
+
store: store2,
|
|
25837
25841
|
methods,
|
|
25838
25842
|
cacheByDefault,
|
|
25839
25843
|
type
|
|
@@ -25872,7 +25876,7 @@ var require_cache2 = __commonJS({
|
|
|
25872
25876
|
return dispatch(opts2, handler);
|
|
25873
25877
|
}
|
|
25874
25878
|
const cacheKey = makeCacheKey(opts2);
|
|
25875
|
-
const result2 =
|
|
25879
|
+
const result2 = store2.get(cacheKey);
|
|
25876
25880
|
if (result2 && typeof result2.then === "function") {
|
|
25877
25881
|
return result2.then((result3) => handleResult2(
|
|
25878
25882
|
dispatch,
|
|
@@ -25905,7 +25909,7 @@ var require_decompress = __commonJS({
|
|
|
25905
25909
|
"../../node_modules/.pnpm/undici@7.25.0/node_modules/undici/lib/interceptor/decompress.js"(exports, module) {
|
|
25906
25910
|
"use strict";
|
|
25907
25911
|
var { createInflate, createGunzip, createBrotliDecompress, createZstdDecompress } = __require("zlib");
|
|
25908
|
-
var { pipeline } = __require("stream");
|
|
25912
|
+
var { pipeline: pipeline2 } = __require("stream");
|
|
25909
25913
|
var DecoratorHandler = require_decorator_handler();
|
|
25910
25914
|
var { runtimeFeatures } = require_runtime_features();
|
|
25911
25915
|
var supportedEncodings = {
|
|
@@ -26014,7 +26018,7 @@ var require_decompress = __commonJS({
|
|
|
26014
26018
|
#setupMultipleDecompressors(controller) {
|
|
26015
26019
|
const lastDecompressor = this.#decompressors[this.#decompressors.length - 1];
|
|
26016
26020
|
this.#setupDecompressorEvents(lastDecompressor, controller);
|
|
26017
|
-
|
|
26021
|
+
pipeline2(this.#decompressors, (err) => {
|
|
26018
26022
|
if (err) {
|
|
26019
26023
|
super.onResponseError(controller, err);
|
|
26020
26024
|
return;
|
|
@@ -26805,12 +26809,12 @@ var require_sqlite_cache_store = __commonJS({
|
|
|
26805
26809
|
assertCacheValue(value);
|
|
26806
26810
|
let size = 0;
|
|
26807
26811
|
const body = [];
|
|
26808
|
-
const
|
|
26812
|
+
const store2 = this;
|
|
26809
26813
|
return new Writable({
|
|
26810
26814
|
decodeStrings: true,
|
|
26811
26815
|
write(chunk, encoding, callback) {
|
|
26812
26816
|
size += chunk.byteLength;
|
|
26813
|
-
if (size <
|
|
26817
|
+
if (size < store2.#maxEntrySize) {
|
|
26814
26818
|
body.push(chunk);
|
|
26815
26819
|
} else {
|
|
26816
26820
|
this.destroy();
|
|
@@ -26818,7 +26822,7 @@ var require_sqlite_cache_store = __commonJS({
|
|
|
26818
26822
|
callback();
|
|
26819
26823
|
},
|
|
26820
26824
|
final(callback) {
|
|
26821
|
-
|
|
26825
|
+
store2.set(key, { ...value, body });
|
|
26822
26826
|
callback();
|
|
26823
26827
|
}
|
|
26824
26828
|
});
|
|
@@ -28754,7 +28758,7 @@ var require_fetch = __commonJS({
|
|
|
28754
28758
|
subresourceSet
|
|
28755
28759
|
} = require_constants3();
|
|
28756
28760
|
var EE = __require("events");
|
|
28757
|
-
var { Readable: Readable2, pipeline, finished, isErrored, isReadable } = __require("stream");
|
|
28761
|
+
var { Readable: Readable2, pipeline: pipeline2, finished, isErrored, isReadable } = __require("stream");
|
|
28758
28762
|
var { addAbortListener, bufferToLowerCasedHeaderName } = require_util();
|
|
28759
28763
|
var { dataURLProcessor, serializeAMimeType, minimizeSupportedMimeType } = require_data_url();
|
|
28760
28764
|
var { getGlobalDispatcher } = require_global2();
|
|
@@ -29745,7 +29749,7 @@ var require_fetch = __commonJS({
|
|
|
29745
29749
|
status: status2,
|
|
29746
29750
|
statusText,
|
|
29747
29751
|
headersList,
|
|
29748
|
-
body: decoders.length ?
|
|
29752
|
+
body: decoders.length ? pipeline2(this.body, ...decoders, (err) => {
|
|
29749
29753
|
if (err) {
|
|
29750
29754
|
this.onError(err);
|
|
29751
29755
|
}
|
|
@@ -32218,9 +32222,9 @@ var require_sender = __commonJS({
|
|
|
32218
32222
|
}
|
|
32219
32223
|
async #run() {
|
|
32220
32224
|
this.#running = true;
|
|
32221
|
-
const
|
|
32222
|
-
while (!
|
|
32223
|
-
const node =
|
|
32225
|
+
const queue2 = this.#queue;
|
|
32226
|
+
while (!queue2.isEmpty()) {
|
|
32227
|
+
const node = queue2.shift();
|
|
32224
32228
|
if (node.promise !== null) {
|
|
32225
32229
|
await node.promise;
|
|
32226
32230
|
}
|
|
@@ -33379,7 +33383,7 @@ ${value}`;
|
|
|
33379
33383
|
var require_eventsource = __commonJS({
|
|
33380
33384
|
"../../node_modules/.pnpm/undici@7.25.0/node_modules/undici/lib/web/eventsource/eventsource.js"(exports, module) {
|
|
33381
33385
|
"use strict";
|
|
33382
|
-
var { pipeline } = __require("stream");
|
|
33386
|
+
var { pipeline: pipeline2 } = __require("stream");
|
|
33383
33387
|
var { fetching } = require_fetch();
|
|
33384
33388
|
var { makeRequest } = require_request2();
|
|
33385
33389
|
var { webidl } = require_webidl();
|
|
@@ -33537,7 +33541,7 @@ var require_eventsource = __commonJS({
|
|
|
33537
33541
|
));
|
|
33538
33542
|
}
|
|
33539
33543
|
});
|
|
33540
|
-
|
|
33544
|
+
pipeline2(
|
|
33541
33545
|
response.body.stream,
|
|
33542
33546
|
eventSourceStream,
|
|
33543
33547
|
(error) => {
|
|
@@ -33892,11 +33896,13 @@ function expBackoff(attempt) {
|
|
|
33892
33896
|
const i = Math.min(Math.max(attempt, 0), BACKOFF_MS.length - 1);
|
|
33893
33897
|
return BACKOFF_MS[i];
|
|
33894
33898
|
}
|
|
33895
|
-
var INGEST_BATCH_MAX_EVENTS, BACKOFF_MS, BACKSTOP_SCAN_MS;
|
|
33899
|
+
var INGEST_BATCH_MAX_EVENTS, SESSION_DEBOUNCE_MS, FORCE_SHIP_THRESHOLD, BACKOFF_MS, BACKSTOP_SCAN_MS;
|
|
33896
33900
|
var init_config = __esm({
|
|
33897
33901
|
"../../packages/daemon-core/src/config/index.ts"() {
|
|
33898
33902
|
"use strict";
|
|
33899
33903
|
INGEST_BATCH_MAX_EVENTS = 1e3;
|
|
33904
|
+
SESSION_DEBOUNCE_MS = 7e3;
|
|
33905
|
+
FORCE_SHIP_THRESHOLD = 200;
|
|
33900
33906
|
BACKOFF_MS = [1e3, 2500, 5e3, 1e4, 2e4, 6e4];
|
|
33901
33907
|
BACKSTOP_SCAN_MS = 5 * 6e4;
|
|
33902
33908
|
}
|
|
@@ -34886,7 +34892,7 @@ async function fetchDeviceViewLedgerByClaim(claimCode) {
|
|
|
34886
34892
|
}
|
|
34887
34893
|
function ingestClient() {
|
|
34888
34894
|
if (_ingest) return _ingest;
|
|
34889
|
-
const
|
|
34895
|
+
const logger2 = createLogger("daemon.ingest");
|
|
34890
34896
|
_ingest = new IngestClient({
|
|
34891
34897
|
apiUrl: state.apiUrl,
|
|
34892
34898
|
auth: {
|
|
@@ -34902,7 +34908,7 @@ function ingestClient() {
|
|
|
34902
34908
|
}
|
|
34903
34909
|
}
|
|
34904
34910
|
},
|
|
34905
|
-
logger
|
|
34911
|
+
logger: logger2
|
|
34906
34912
|
});
|
|
34907
34913
|
return _ingest;
|
|
34908
34914
|
}
|
|
@@ -35083,17 +35089,101 @@ var init_ids2 = __esm({
|
|
|
35083
35089
|
});
|
|
35084
35090
|
|
|
35085
35091
|
// ../../packages/daemon-core/src/queue/index.ts
|
|
35092
|
+
async function buildBatches(opts) {
|
|
35093
|
+
const debounceMs = opts.debounceMs ?? SESSION_DEBOUNCE_MS;
|
|
35094
|
+
const maxEvents = opts.maxEvents ?? INGEST_BATCH_MAX_EVENTS;
|
|
35095
|
+
const forceThreshold = opts.forceShipThreshold ?? FORCE_SHIP_THRESHOLD;
|
|
35096
|
+
const bySession = await opts.store.listUnsentBySession();
|
|
35097
|
+
const batches = [];
|
|
35098
|
+
let eventsInCurrentBatch = [];
|
|
35099
|
+
let sessionIdsInCurrentBatch = [];
|
|
35100
|
+
let toolCallsInCurrentBatch = [];
|
|
35101
|
+
for (const [, items] of bySession) {
|
|
35102
|
+
if (items.length === 0) continue;
|
|
35103
|
+
const lastTs = Math.max(...items.map((i) => i.last_event_ts_ms));
|
|
35104
|
+
const quiet = opts.nowMs - lastTs >= debounceMs;
|
|
35105
|
+
if (!quiet && items.length < forceThreshold) continue;
|
|
35106
|
+
for (const item of items) {
|
|
35107
|
+
const itemCalls = item.tool_calls ?? [];
|
|
35108
|
+
if (eventsInCurrentBatch.length >= maxEvents || eventsInCurrentBatch.length > 0 && toolCallsInCurrentBatch.length + itemCalls.length > INGEST_BATCH_MAX_TOOL_CALLS) {
|
|
35109
|
+
batches.push(
|
|
35110
|
+
await finalise({
|
|
35111
|
+
events: eventsInCurrentBatch,
|
|
35112
|
+
sessionIds: sessionIdsInCurrentBatch,
|
|
35113
|
+
toolCalls: toolCallsInCurrentBatch,
|
|
35114
|
+
pipeline: opts.pipeline,
|
|
35115
|
+
deviceId: opts.deviceId,
|
|
35116
|
+
daemonVersion: opts.daemonVersion
|
|
35117
|
+
})
|
|
35118
|
+
);
|
|
35119
|
+
eventsInCurrentBatch = [];
|
|
35120
|
+
sessionIdsInCurrentBatch = [];
|
|
35121
|
+
toolCallsInCurrentBatch = [];
|
|
35122
|
+
}
|
|
35123
|
+
eventsInCurrentBatch.push(item.event);
|
|
35124
|
+
sessionIdsInCurrentBatch.push(item.session_id);
|
|
35125
|
+
toolCallsInCurrentBatch.push(...itemCalls);
|
|
35126
|
+
}
|
|
35127
|
+
}
|
|
35128
|
+
if (eventsInCurrentBatch.length > 0) {
|
|
35129
|
+
batches.push(
|
|
35130
|
+
await finalise({
|
|
35131
|
+
events: eventsInCurrentBatch,
|
|
35132
|
+
sessionIds: sessionIdsInCurrentBatch,
|
|
35133
|
+
toolCalls: toolCallsInCurrentBatch,
|
|
35134
|
+
pipeline: opts.pipeline,
|
|
35135
|
+
deviceId: opts.deviceId,
|
|
35136
|
+
daemonVersion: opts.daemonVersion
|
|
35137
|
+
})
|
|
35138
|
+
);
|
|
35139
|
+
}
|
|
35140
|
+
return batches;
|
|
35141
|
+
}
|
|
35142
|
+
function attachSegmentIds(calls, segments) {
|
|
35143
|
+
const segmentByEvent = /* @__PURE__ */ new Map();
|
|
35144
|
+
for (const seg of segments) {
|
|
35145
|
+
for (const id of seg.source_event_ids) segmentByEvent.set(id, seg.segment_id);
|
|
35146
|
+
}
|
|
35147
|
+
return attachSegmentIdsByMap(calls, segmentByEvent);
|
|
35148
|
+
}
|
|
35086
35149
|
function attachSegmentIdsByMap(calls, segmentByEvent) {
|
|
35087
35150
|
return calls.map((c) => ({
|
|
35088
35151
|
...c,
|
|
35089
35152
|
segment_id: segmentByEvent.get(c.source_event_id) ?? null
|
|
35090
35153
|
}));
|
|
35091
35154
|
}
|
|
35155
|
+
async function finalise(args) {
|
|
35156
|
+
const segments = [];
|
|
35157
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
35158
|
+
for (let i = 0; i < args.events.length; i++) {
|
|
35159
|
+
const sid = args.sessionIds[i] ?? args.events[i]?.session_id ?? "unknown";
|
|
35160
|
+
const arr = bySession.get(sid) ?? [];
|
|
35161
|
+
const ev = args.events[i];
|
|
35162
|
+
if (ev) arr.push(ev);
|
|
35163
|
+
bySession.set(sid, arr);
|
|
35164
|
+
}
|
|
35165
|
+
for (const [, sessionEvents] of bySession) {
|
|
35166
|
+
const segs = await args.pipeline.run(sessionEvents);
|
|
35167
|
+
segments.push(...segs);
|
|
35168
|
+
}
|
|
35169
|
+
return {
|
|
35170
|
+
batch_id: batchId(),
|
|
35171
|
+
device_id: args.deviceId,
|
|
35172
|
+
daemon_version: args.daemonVersion,
|
|
35173
|
+
events: args.events,
|
|
35174
|
+
segments,
|
|
35175
|
+
// Segments are in hand here, so segment attribution happens at the
|
|
35176
|
+
// last responsible moment before the wire.
|
|
35177
|
+
tool_calls: attachSegmentIds(args.toolCalls, segments)
|
|
35178
|
+
};
|
|
35179
|
+
}
|
|
35180
|
+
var INGEST_BATCH_MAX_TOOL_CALLS;
|
|
35092
35181
|
var init_queue = __esm({
|
|
35093
35182
|
"../../packages/daemon-core/src/queue/index.ts"() {
|
|
35094
35183
|
"use strict";
|
|
35095
35184
|
init_config();
|
|
35096
35185
|
init_ids2();
|
|
35186
|
+
INGEST_BATCH_MAX_TOOL_CALLS = 2e4;
|
|
35097
35187
|
}
|
|
35098
35188
|
});
|
|
35099
35189
|
|
|
@@ -37086,7 +37176,7 @@ var init_scan = __esm({
|
|
|
37086
37176
|
init_api();
|
|
37087
37177
|
init_config2();
|
|
37088
37178
|
init_pipeline2();
|
|
37089
|
-
DAEMON_VERSION = true ? "daemon-0.
|
|
37179
|
+
DAEMON_VERSION = true ? "daemon-0.8.0" : "daemon-dev";
|
|
37090
37180
|
BATCH_MAX_EVENTS = INGEST_BATCH_MAX_EVENTS;
|
|
37091
37181
|
BATCH_MAX_TOOL_CALLS = 2e4;
|
|
37092
37182
|
BATCH_BUFFER_HARD_CAP = BATCH_MAX_EVENTS * 2;
|
|
@@ -37323,6 +37413,165 @@ var init_update = __esm({
|
|
|
37323
37413
|
}
|
|
37324
37414
|
});
|
|
37325
37415
|
|
|
37416
|
+
// src/receiver.ts
|
|
37417
|
+
import { createServer } from "http";
|
|
37418
|
+
function queue() {
|
|
37419
|
+
if (!store) store = new FileQueueStore(homePath("sdk-ingest-queue.json"));
|
|
37420
|
+
return store;
|
|
37421
|
+
}
|
|
37422
|
+
function parseBatch(json) {
|
|
37423
|
+
if (typeof json !== "object" || json === null) return { error: "body must be a JSON object" };
|
|
37424
|
+
const b = json;
|
|
37425
|
+
if (!Array.isArray(b.events) || b.events.length === 0)
|
|
37426
|
+
return { error: "events must be a non-empty array" };
|
|
37427
|
+
if (b.events.length > 1e4) return { error: "too many events (max 10000)" };
|
|
37428
|
+
for (const e of b.events) {
|
|
37429
|
+
if (typeof e !== "object" || e === null) return { error: "each event must be an object" };
|
|
37430
|
+
const ev = e;
|
|
37431
|
+
for (const k of ["source_event_id", "session_id", "agent", "ts"]) {
|
|
37432
|
+
if (typeof ev[k] !== "string" || ev[k].length === 0)
|
|
37433
|
+
return { error: `event.${k} is required` };
|
|
37434
|
+
}
|
|
37435
|
+
}
|
|
37436
|
+
const toolCalls = Array.isArray(b.tool_calls) ? b.tool_calls : [];
|
|
37437
|
+
return { events: b.events, toolCalls };
|
|
37438
|
+
}
|
|
37439
|
+
async function enqueue(batch) {
|
|
37440
|
+
const q = queue();
|
|
37441
|
+
for (const event of batch.events) {
|
|
37442
|
+
const calls = batch.toolCalls.filter((tc) => tc.source_event_id === event.source_event_id).map(({ segment_id: _segmentId, ...draft }) => draft);
|
|
37443
|
+
await q.put({
|
|
37444
|
+
source_event_id: event.source_event_id,
|
|
37445
|
+
session_id: event.session_id,
|
|
37446
|
+
agent: event.agent,
|
|
37447
|
+
event,
|
|
37448
|
+
last_event_ts_ms: Date.parse(event.ts) || Date.now(),
|
|
37449
|
+
synced: false,
|
|
37450
|
+
sent_batch_id: null,
|
|
37451
|
+
tool_calls: calls.length > 0 ? calls : void 0
|
|
37452
|
+
});
|
|
37453
|
+
}
|
|
37454
|
+
return batch.events.length;
|
|
37455
|
+
}
|
|
37456
|
+
async function drainLocalQueue(opts) {
|
|
37457
|
+
if (draining) return { batches: 0, events: 0 };
|
|
37458
|
+
draining = true;
|
|
37459
|
+
try {
|
|
37460
|
+
const q = queue();
|
|
37461
|
+
if (await q.countUnsent() === 0) return { batches: 0, events: 0 };
|
|
37462
|
+
const batches = await buildBatches({
|
|
37463
|
+
store: q,
|
|
37464
|
+
pipeline,
|
|
37465
|
+
deviceId: opts.deviceId,
|
|
37466
|
+
daemonVersion: opts.daemonVersion,
|
|
37467
|
+
nowMs: Date.now()
|
|
37468
|
+
});
|
|
37469
|
+
let events = 0;
|
|
37470
|
+
for (const batch of batches) {
|
|
37471
|
+
const shipped = {
|
|
37472
|
+
...batch,
|
|
37473
|
+
events: batch.events.map(({ content_excerpt: _excerpt, ...rest }) => rest)
|
|
37474
|
+
};
|
|
37475
|
+
await uploadBatch(shipped);
|
|
37476
|
+
await q.markSent(
|
|
37477
|
+
batch.events.map((e) => e.source_event_id),
|
|
37478
|
+
batch.batch_id
|
|
37479
|
+
);
|
|
37480
|
+
events += batch.events.length;
|
|
37481
|
+
}
|
|
37482
|
+
return { batches: batches.length, events };
|
|
37483
|
+
} finally {
|
|
37484
|
+
draining = false;
|
|
37485
|
+
}
|
|
37486
|
+
}
|
|
37487
|
+
function localQueueDepth() {
|
|
37488
|
+
return queue().countUnsent();
|
|
37489
|
+
}
|
|
37490
|
+
function startLocalIngestReceiver(opts = {}) {
|
|
37491
|
+
const port = opts.port ?? (Number(process.env.MODELSTAT_LOCAL_INGEST_PORT) || DEFAULT_LOCAL_INGEST_PORT);
|
|
37492
|
+
return new Promise((resolve6) => {
|
|
37493
|
+
const server = createServer((req, res) => void handle(req, res));
|
|
37494
|
+
let settled = false;
|
|
37495
|
+
server.on("error", (err) => {
|
|
37496
|
+
if (settled) return;
|
|
37497
|
+
settled = true;
|
|
37498
|
+
logger.warn(
|
|
37499
|
+
`local ingest receiver disabled \u2014 SDK local_daemon mode unavailable: ${err.code ?? err.message} on 127.0.0.1:${port}`
|
|
37500
|
+
);
|
|
37501
|
+
resolve6(null);
|
|
37502
|
+
});
|
|
37503
|
+
server.listen(port, "127.0.0.1", () => {
|
|
37504
|
+
settled = true;
|
|
37505
|
+
const addr = server.address();
|
|
37506
|
+
const boundPort = typeof addr === "object" && addr ? addr.port : port;
|
|
37507
|
+
logger.info(`local ingest receiver on http://127.0.0.1:${boundPort}/v1/ingest`);
|
|
37508
|
+
resolve6({
|
|
37509
|
+
port: boundPort,
|
|
37510
|
+
close: () => new Promise((r) => server.close(() => r()))
|
|
37511
|
+
});
|
|
37512
|
+
});
|
|
37513
|
+
});
|
|
37514
|
+
}
|
|
37515
|
+
async function handle(req, res) {
|
|
37516
|
+
const send = (code, body) => {
|
|
37517
|
+
res.writeHead(code, { "content-type": "application/json" });
|
|
37518
|
+
res.end(JSON.stringify(body));
|
|
37519
|
+
};
|
|
37520
|
+
const path = (req.url ?? "").split("?")[0];
|
|
37521
|
+
if (req.method === "GET" && path === "/healthz") return send(200, { ok: true });
|
|
37522
|
+
if (req.method !== "POST" || path !== "/v1/ingest" && path !== "/v1/ingest/raw") {
|
|
37523
|
+
return send(404, { error: "not found" });
|
|
37524
|
+
}
|
|
37525
|
+
let size = 0;
|
|
37526
|
+
const chunks = [];
|
|
37527
|
+
try {
|
|
37528
|
+
for await (const chunk of req) {
|
|
37529
|
+
size += chunk.length;
|
|
37530
|
+
if (size > MAX_BODY_BYTES) {
|
|
37531
|
+
send(413, { error: "batch too large" });
|
|
37532
|
+
req.destroy();
|
|
37533
|
+
return;
|
|
37534
|
+
}
|
|
37535
|
+
chunks.push(chunk);
|
|
37536
|
+
}
|
|
37537
|
+
} catch {
|
|
37538
|
+
return send(400, { error: "read error" });
|
|
37539
|
+
}
|
|
37540
|
+
let json;
|
|
37541
|
+
try {
|
|
37542
|
+
json = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
37543
|
+
} catch {
|
|
37544
|
+
return send(400, { error: "invalid json" });
|
|
37545
|
+
}
|
|
37546
|
+
const parsed = parseBatch(json);
|
|
37547
|
+
if ("error" in parsed) return send(400, parsed);
|
|
37548
|
+
try {
|
|
37549
|
+
const accepted = await enqueue(parsed);
|
|
37550
|
+
return send(200, { accepted, queued: true });
|
|
37551
|
+
} catch (e) {
|
|
37552
|
+
logger.warn(`local ingest enqueue failed: ${e.message}`);
|
|
37553
|
+
return send(500, { error: "enqueue failed" });
|
|
37554
|
+
}
|
|
37555
|
+
}
|
|
37556
|
+
var logger, DEFAULT_LOCAL_INGEST_PORT, MAX_BODY_BYTES, store, pipeline, draining;
|
|
37557
|
+
var init_receiver = __esm({
|
|
37558
|
+
"src/receiver.ts"() {
|
|
37559
|
+
"use strict";
|
|
37560
|
+
init_logger();
|
|
37561
|
+
init_node();
|
|
37562
|
+
init_queue();
|
|
37563
|
+
init_api();
|
|
37564
|
+
init_paths();
|
|
37565
|
+
init_pipeline2();
|
|
37566
|
+
logger = createLogger("daemon.local-ingest");
|
|
37567
|
+
DEFAULT_LOCAL_INGEST_PORT = 4319;
|
|
37568
|
+
MAX_BODY_BYTES = 16 * 1024 * 1024;
|
|
37569
|
+
store = null;
|
|
37570
|
+
pipeline = { run: (events) => buildSegments(events) };
|
|
37571
|
+
draining = false;
|
|
37572
|
+
}
|
|
37573
|
+
});
|
|
37574
|
+
|
|
37326
37575
|
// src/reconcile.ts
|
|
37327
37576
|
import { stat as stat3 } from "fs/promises";
|
|
37328
37577
|
function utcDay(ts) {
|
|
@@ -37700,19 +37949,19 @@ function createPolicyRefresher(opts) {
|
|
|
37700
37949
|
...opts.fetch ? { fetch: opts.fetch } : {},
|
|
37701
37950
|
...opts.logger ? { logger: opts.logger } : {}
|
|
37702
37951
|
};
|
|
37703
|
-
const
|
|
37952
|
+
const store2 = new RemoteConfigStore(env, [policiesKind]);
|
|
37704
37953
|
let timer = null;
|
|
37705
37954
|
const apply = () => {
|
|
37706
|
-
const bundle =
|
|
37955
|
+
const bundle = store2.get("policies");
|
|
37707
37956
|
setRemoteRedactionPatterns(compilePolicyPatterns(bundle));
|
|
37708
37957
|
};
|
|
37709
37958
|
const refresh = async () => {
|
|
37710
|
-
await
|
|
37959
|
+
await store2.refresh("policies");
|
|
37711
37960
|
apply();
|
|
37712
37961
|
};
|
|
37713
37962
|
return {
|
|
37714
37963
|
async start() {
|
|
37715
|
-
await
|
|
37964
|
+
await store2.initFromCache();
|
|
37716
37965
|
apply();
|
|
37717
37966
|
void refresh().catch(() => {
|
|
37718
37967
|
});
|
|
@@ -39739,6 +39988,25 @@ async function runDaemon(opts = {}) {
|
|
|
39739
39988
|
setPhase("error", `summariser preflight failed: ${err.message}`);
|
|
39740
39989
|
throw err;
|
|
39741
39990
|
}
|
|
39991
|
+
const localIngest = await startLocalIngestReceiver();
|
|
39992
|
+
const LOCAL_DRAIN_INTERVAL_MS = 5e3;
|
|
39993
|
+
let localDrainTimer = null;
|
|
39994
|
+
if (localIngest) {
|
|
39995
|
+
const drainTick = async () => {
|
|
39996
|
+
try {
|
|
39997
|
+
const { events } = await drainLocalQueue({
|
|
39998
|
+
deviceId: state.deviceId,
|
|
39999
|
+
daemonVersion: DAEMON_VERSION2
|
|
40000
|
+
});
|
|
40001
|
+
if (events > 0) bumpStat("sdk_events_uploaded", events);
|
|
40002
|
+
setStat("sdk_queue", await localQueueDepth());
|
|
40003
|
+
} catch (e) {
|
|
40004
|
+
setMessage(`SDK ingest upload deferred: ${describeErrorWithCause(e)}`);
|
|
40005
|
+
}
|
|
40006
|
+
};
|
|
40007
|
+
localDrainTimer = setInterval(() => void drainTick(), LOCAL_DRAIN_INTERVAL_MS);
|
|
40008
|
+
localDrainTimer.unref();
|
|
40009
|
+
}
|
|
39742
40010
|
const { reconcileProcessingVersion: reconcileProcessingVersion2 } = await Promise.resolve().then(() => (init_processing_version(), processing_version_exports));
|
|
39743
40011
|
const pv = reconcileProcessingVersion2(state);
|
|
39744
40012
|
if (pv.changed) {
|
|
@@ -39805,6 +40073,8 @@ async function runDaemon(opts = {}) {
|
|
|
39805
40073
|
setPhase("offline", "Shutting down");
|
|
39806
40074
|
await sendHeartbeat();
|
|
39807
40075
|
await watcher.close();
|
|
40076
|
+
if (localDrainTimer) clearInterval(localDrainTimer);
|
|
40077
|
+
await localIngest?.close();
|
|
39808
40078
|
try {
|
|
39809
40079
|
const { disposeLlama: disposeLlama2 } = await Promise.resolve().then(() => (init_node(), node_exports));
|
|
39810
40080
|
await disposeLlama2();
|
|
@@ -39828,11 +40098,12 @@ var init_daemon = __esm({
|
|
|
39828
40098
|
init_config2();
|
|
39829
40099
|
init_lock();
|
|
39830
40100
|
init_machine_key();
|
|
40101
|
+
init_receiver();
|
|
39831
40102
|
init_reconcile();
|
|
39832
40103
|
init_scan();
|
|
39833
40104
|
init_single_flight();
|
|
39834
40105
|
init_update();
|
|
39835
|
-
DAEMON_VERSION2 = true ? "daemon-0.
|
|
40106
|
+
DAEMON_VERSION2 = true ? "daemon-0.8.0" : "daemon-dev";
|
|
39836
40107
|
HEARTBEAT_INTERVAL_MS = 1e4;
|
|
39837
40108
|
SCAN_INTERVAL_MS = 5 * 60 * 1e3;
|
|
39838
40109
|
DISCOVERY_INTERVAL_MS = 6e4;
|
|
@@ -40436,7 +40707,7 @@ function tryOpenBrowser(url) {
|
|
|
40436
40707
|
return false;
|
|
40437
40708
|
}
|
|
40438
40709
|
}
|
|
40439
|
-
var DAEMON_VERSION3 = true ? "daemon-0.
|
|
40710
|
+
var DAEMON_VERSION3 = true ? "daemon-0.8.0" : "daemon-dev";
|
|
40440
40711
|
function osFamily() {
|
|
40441
40712
|
const p = platform6();
|
|
40442
40713
|
if (p === "darwin") return "macos";
|