modelstat 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +332 -65
- package/dist/cli.mjs.map +1 -1
- package/package.json +3 -3
package/dist/cli.mjs
CHANGED
|
@@ -16734,7 +16734,7 @@ var require_client_h2 = __commonJS({
|
|
|
16734
16734
|
"../../node_modules/.pnpm/undici@7.25.0/node_modules/undici/lib/dispatcher/client-h2.js"(exports, module) {
|
|
16735
16735
|
"use strict";
|
|
16736
16736
|
var assert = __require("assert");
|
|
16737
|
-
var { pipeline } = __require("stream");
|
|
16737
|
+
var { pipeline: pipeline2 } = __require("stream");
|
|
16738
16738
|
var util2 = require_util();
|
|
16739
16739
|
var {
|
|
16740
16740
|
RequestContentLengthMismatchError,
|
|
@@ -17367,7 +17367,7 @@ var require_client_h2 = __commonJS({
|
|
|
17367
17367
|
}
|
|
17368
17368
|
function writeStream(abort, socket, expectsPayload, h2stream, body, client, request3, contentLength) {
|
|
17369
17369
|
assert(contentLength !== 0 || client[kRunning] === 0, "stream body cannot be pipelined");
|
|
17370
|
-
const pipe =
|
|
17370
|
+
const pipe = pipeline2(
|
|
17371
17371
|
body,
|
|
17372
17372
|
h2stream,
|
|
17373
17373
|
(err) => {
|
|
@@ -18074,10 +18074,10 @@ var require_pool_base = __commonJS({
|
|
|
18074
18074
|
[kClients] = [];
|
|
18075
18075
|
[kNeedDrain] = false;
|
|
18076
18076
|
[kOnDrain](client, origin, targets) {
|
|
18077
|
-
const
|
|
18077
|
+
const queue2 = this[kQueue];
|
|
18078
18078
|
let needDrain = false;
|
|
18079
18079
|
while (!needDrain) {
|
|
18080
|
-
const item =
|
|
18080
|
+
const item = queue2.shift();
|
|
18081
18081
|
if (!item) {
|
|
18082
18082
|
break;
|
|
18083
18083
|
}
|
|
@@ -18089,7 +18089,7 @@ var require_pool_base = __commonJS({
|
|
|
18089
18089
|
this[kNeedDrain] = false;
|
|
18090
18090
|
this.emit("drain", origin, [this, ...targets]);
|
|
18091
18091
|
}
|
|
18092
|
-
if (this[kClosedResolve] &&
|
|
18092
|
+
if (this[kClosedResolve] && queue2.isEmpty()) {
|
|
18093
18093
|
const closeAll = [];
|
|
18094
18094
|
for (let i = 0; i < this[kClients].length; i++) {
|
|
18095
18095
|
const client2 = this[kClients][i];
|
|
@@ -21133,7 +21133,7 @@ var require_api_pipeline = __commonJS({
|
|
|
21133
21133
|
util2.destroy(ret, err);
|
|
21134
21134
|
}
|
|
21135
21135
|
};
|
|
21136
|
-
function
|
|
21136
|
+
function pipeline2(opts, handler) {
|
|
21137
21137
|
try {
|
|
21138
21138
|
const pipelineHandler = new PipelineHandler(opts, handler);
|
|
21139
21139
|
this.dispatch({ ...opts, body: pipelineHandler.req }, pipelineHandler);
|
|
@@ -21142,7 +21142,7 @@ var require_api_pipeline = __commonJS({
|
|
|
21142
21142
|
return new PassThrough().destroy(err);
|
|
21143
21143
|
}
|
|
21144
21144
|
}
|
|
21145
|
-
module.exports =
|
|
21145
|
+
module.exports = pipeline2;
|
|
21146
21146
|
}
|
|
21147
21147
|
});
|
|
21148
21148
|
|
|
@@ -22014,13 +22014,13 @@ var require_mock_call_history = __commonJS({
|
|
|
22014
22014
|
"use strict";
|
|
22015
22015
|
var { kMockCallHistoryAddLog } = require_mock_symbols();
|
|
22016
22016
|
var { InvalidArgumentError } = require_errors();
|
|
22017
|
-
function handleFilterCallsWithOptions(criteria, options, handler,
|
|
22017
|
+
function handleFilterCallsWithOptions(criteria, options, handler, store2) {
|
|
22018
22018
|
switch (options.operator) {
|
|
22019
22019
|
case "OR":
|
|
22020
|
-
|
|
22021
|
-
return
|
|
22020
|
+
store2.push(...handler(criteria));
|
|
22021
|
+
return store2;
|
|
22022
22022
|
case "AND":
|
|
22023
|
-
return handler.call({ logs:
|
|
22023
|
+
return handler.call({ logs: store2 }, criteria);
|
|
22024
22024
|
default:
|
|
22025
22025
|
throw new InvalidArgumentError("options.operator must to be a case insensitive string equal to 'OR' or 'AND'");
|
|
22026
22026
|
}
|
|
@@ -24377,12 +24377,12 @@ var require_cache = __commonJS({
|
|
|
24377
24377
|
}
|
|
24378
24378
|
return false;
|
|
24379
24379
|
}
|
|
24380
|
-
function assertCacheStore(
|
|
24381
|
-
if (typeof
|
|
24382
|
-
throw new TypeError(`expected type of ${name} to be a CacheStore, got ${
|
|
24380
|
+
function assertCacheStore(store2, name = "CacheStore") {
|
|
24381
|
+
if (typeof store2 !== "object" || store2 === null) {
|
|
24382
|
+
throw new TypeError(`expected type of ${name} to be a CacheStore, got ${store2 === null ? "null" : typeof store2}`);
|
|
24383
24383
|
}
|
|
24384
24384
|
for (const fn of ["get", "createWriteStream", "delete"]) {
|
|
24385
|
-
if (typeof
|
|
24385
|
+
if (typeof store2[fn] !== "function") {
|
|
24386
24386
|
throw new TypeError(`${name} needs to have a \`${fn}()\` function`);
|
|
24387
24387
|
}
|
|
24388
24388
|
}
|
|
@@ -24986,8 +24986,8 @@ var require_cache_handler = __commonJS({
|
|
|
24986
24986
|
* @param {import('../../types/cache-interceptor.d.ts').default.CacheKey} cacheKey
|
|
24987
24987
|
* @param {import('../../types/dispatcher.d.ts').default.DispatchHandler} handler
|
|
24988
24988
|
*/
|
|
24989
|
-
constructor({ store, type, cacheByDefault }, cacheKey, handler) {
|
|
24990
|
-
this.#store =
|
|
24989
|
+
constructor({ store: store2, type, cacheByDefault }, cacheKey, handler) {
|
|
24990
|
+
this.#store = store2;
|
|
24991
24991
|
this.#cacheType = type;
|
|
24992
24992
|
this.#cacheByDefault = cacheByDefault;
|
|
24993
24993
|
this.#cacheKey = cacheKey;
|
|
@@ -25396,7 +25396,7 @@ var require_memory_cache_store = __commonJS({
|
|
|
25396
25396
|
assertCacheKey(key);
|
|
25397
25397
|
assertCacheValue(val);
|
|
25398
25398
|
const topLevelKey = `${key.origin}:${key.path}`;
|
|
25399
|
-
const
|
|
25399
|
+
const store2 = this;
|
|
25400
25400
|
const entry = { ...key, ...val, body: [], size: 0 };
|
|
25401
25401
|
return new Writable({
|
|
25402
25402
|
write(chunk, encoding, callback) {
|
|
@@ -25404,7 +25404,7 @@ var require_memory_cache_store = __commonJS({
|
|
|
25404
25404
|
chunk = Buffer.from(chunk, encoding);
|
|
25405
25405
|
}
|
|
25406
25406
|
entry.size += chunk.byteLength;
|
|
25407
|
-
if (entry.size >=
|
|
25407
|
+
if (entry.size >= store2.#maxEntrySize) {
|
|
25408
25408
|
this.destroy();
|
|
25409
25409
|
} else {
|
|
25410
25410
|
entry.body.push(chunk);
|
|
@@ -25412,42 +25412,42 @@ var require_memory_cache_store = __commonJS({
|
|
|
25412
25412
|
callback(null);
|
|
25413
25413
|
},
|
|
25414
25414
|
final(callback) {
|
|
25415
|
-
let entries =
|
|
25415
|
+
let entries = store2.#entries.get(topLevelKey);
|
|
25416
25416
|
if (!entries) {
|
|
25417
25417
|
entries = [];
|
|
25418
|
-
|
|
25418
|
+
store2.#entries.set(topLevelKey, entries);
|
|
25419
25419
|
}
|
|
25420
25420
|
const previousEntry = findEntry(key, entries, Date.now());
|
|
25421
25421
|
if (previousEntry) {
|
|
25422
25422
|
const index = entries.indexOf(previousEntry);
|
|
25423
25423
|
entries.splice(index, 1, entry);
|
|
25424
|
-
|
|
25424
|
+
store2.#size -= previousEntry.size;
|
|
25425
25425
|
} else {
|
|
25426
25426
|
entries.push(entry);
|
|
25427
|
-
|
|
25428
|
-
}
|
|
25429
|
-
|
|
25430
|
-
if (
|
|
25431
|
-
if (!
|
|
25432
|
-
|
|
25433
|
-
size:
|
|
25434
|
-
maxSize:
|
|
25435
|
-
count:
|
|
25436
|
-
maxCount:
|
|
25427
|
+
store2.#count += 1;
|
|
25428
|
+
}
|
|
25429
|
+
store2.#size += entry.size;
|
|
25430
|
+
if (store2.#size > store2.#maxSize || store2.#count > store2.#maxCount) {
|
|
25431
|
+
if (!store2.#hasEmittedMaxSizeEvent) {
|
|
25432
|
+
store2.emit("maxSizeExceeded", {
|
|
25433
|
+
size: store2.#size,
|
|
25434
|
+
maxSize: store2.#maxSize,
|
|
25435
|
+
count: store2.#count,
|
|
25436
|
+
maxCount: store2.#maxCount
|
|
25437
25437
|
});
|
|
25438
|
-
|
|
25438
|
+
store2.#hasEmittedMaxSizeEvent = true;
|
|
25439
25439
|
}
|
|
25440
|
-
for (const [key2, entries2] of
|
|
25440
|
+
for (const [key2, entries2] of store2.#entries) {
|
|
25441
25441
|
for (const entry2 of entries2.splice(0, entries2.length / 2)) {
|
|
25442
|
-
|
|
25443
|
-
|
|
25442
|
+
store2.#size -= entry2.size;
|
|
25443
|
+
store2.#count -= 1;
|
|
25444
25444
|
}
|
|
25445
25445
|
if (entries2.length === 0) {
|
|
25446
|
-
|
|
25446
|
+
store2.#entries.delete(key2);
|
|
25447
25447
|
}
|
|
25448
25448
|
}
|
|
25449
|
-
if (
|
|
25450
|
-
|
|
25449
|
+
if (store2.#size < store2.#maxSize && store2.#count < store2.#maxCount) {
|
|
25450
|
+
store2.#hasEmittedMaxSizeEvent = false;
|
|
25451
25451
|
}
|
|
25452
25452
|
}
|
|
25453
25453
|
callback(null);
|
|
@@ -25818,7 +25818,7 @@ var require_cache2 = __commonJS({
|
|
|
25818
25818
|
}
|
|
25819
25819
|
module.exports = (opts = {}) => {
|
|
25820
25820
|
const {
|
|
25821
|
-
store = new MemoryCacheStore(),
|
|
25821
|
+
store: store2 = new MemoryCacheStore(),
|
|
25822
25822
|
methods = ["GET"],
|
|
25823
25823
|
cacheByDefault = void 0,
|
|
25824
25824
|
type = "shared",
|
|
@@ -25827,7 +25827,7 @@ var require_cache2 = __commonJS({
|
|
|
25827
25827
|
if (typeof opts !== "object" || opts === null) {
|
|
25828
25828
|
throw new TypeError(`expected type of opts to be an Object, got ${opts === null ? "null" : typeof opts}`);
|
|
25829
25829
|
}
|
|
25830
|
-
assertCacheStore(
|
|
25830
|
+
assertCacheStore(store2, "opts.store");
|
|
25831
25831
|
assertCacheMethods(methods, "opts.methods");
|
|
25832
25832
|
assertCacheOrigins(origins, "opts.origins");
|
|
25833
25833
|
if (typeof cacheByDefault !== "undefined" && typeof cacheByDefault !== "number") {
|
|
@@ -25837,7 +25837,7 @@ var require_cache2 = __commonJS({
|
|
|
25837
25837
|
throw new TypeError(`expected opts.type to be shared, private, or undefined, got ${typeof type}`);
|
|
25838
25838
|
}
|
|
25839
25839
|
const globalOpts = {
|
|
25840
|
-
store,
|
|
25840
|
+
store: store2,
|
|
25841
25841
|
methods,
|
|
25842
25842
|
cacheByDefault,
|
|
25843
25843
|
type
|
|
@@ -25876,7 +25876,7 @@ var require_cache2 = __commonJS({
|
|
|
25876
25876
|
return dispatch(opts2, handler);
|
|
25877
25877
|
}
|
|
25878
25878
|
const cacheKey = makeCacheKey(opts2);
|
|
25879
|
-
const result2 =
|
|
25879
|
+
const result2 = store2.get(cacheKey);
|
|
25880
25880
|
if (result2 && typeof result2.then === "function") {
|
|
25881
25881
|
return result2.then((result3) => handleResult2(
|
|
25882
25882
|
dispatch,
|
|
@@ -25909,7 +25909,7 @@ var require_decompress = __commonJS({
|
|
|
25909
25909
|
"../../node_modules/.pnpm/undici@7.25.0/node_modules/undici/lib/interceptor/decompress.js"(exports, module) {
|
|
25910
25910
|
"use strict";
|
|
25911
25911
|
var { createInflate, createGunzip, createBrotliDecompress, createZstdDecompress } = __require("zlib");
|
|
25912
|
-
var { pipeline } = __require("stream");
|
|
25912
|
+
var { pipeline: pipeline2 } = __require("stream");
|
|
25913
25913
|
var DecoratorHandler = require_decorator_handler();
|
|
25914
25914
|
var { runtimeFeatures } = require_runtime_features();
|
|
25915
25915
|
var supportedEncodings = {
|
|
@@ -26018,7 +26018,7 @@ var require_decompress = __commonJS({
|
|
|
26018
26018
|
#setupMultipleDecompressors(controller) {
|
|
26019
26019
|
const lastDecompressor = this.#decompressors[this.#decompressors.length - 1];
|
|
26020
26020
|
this.#setupDecompressorEvents(lastDecompressor, controller);
|
|
26021
|
-
|
|
26021
|
+
pipeline2(this.#decompressors, (err) => {
|
|
26022
26022
|
if (err) {
|
|
26023
26023
|
super.onResponseError(controller, err);
|
|
26024
26024
|
return;
|
|
@@ -26809,12 +26809,12 @@ var require_sqlite_cache_store = __commonJS({
|
|
|
26809
26809
|
assertCacheValue(value);
|
|
26810
26810
|
let size = 0;
|
|
26811
26811
|
const body = [];
|
|
26812
|
-
const
|
|
26812
|
+
const store2 = this;
|
|
26813
26813
|
return new Writable({
|
|
26814
26814
|
decodeStrings: true,
|
|
26815
26815
|
write(chunk, encoding, callback) {
|
|
26816
26816
|
size += chunk.byteLength;
|
|
26817
|
-
if (size <
|
|
26817
|
+
if (size < store2.#maxEntrySize) {
|
|
26818
26818
|
body.push(chunk);
|
|
26819
26819
|
} else {
|
|
26820
26820
|
this.destroy();
|
|
@@ -26822,7 +26822,7 @@ var require_sqlite_cache_store = __commonJS({
|
|
|
26822
26822
|
callback();
|
|
26823
26823
|
},
|
|
26824
26824
|
final(callback) {
|
|
26825
|
-
|
|
26825
|
+
store2.set(key, { ...value, body });
|
|
26826
26826
|
callback();
|
|
26827
26827
|
}
|
|
26828
26828
|
});
|
|
@@ -28758,7 +28758,7 @@ var require_fetch = __commonJS({
|
|
|
28758
28758
|
subresourceSet
|
|
28759
28759
|
} = require_constants3();
|
|
28760
28760
|
var EE = __require("events");
|
|
28761
|
-
var { Readable: Readable2, pipeline, finished, isErrored, isReadable } = __require("stream");
|
|
28761
|
+
var { Readable: Readable2, pipeline: pipeline2, finished, isErrored, isReadable } = __require("stream");
|
|
28762
28762
|
var { addAbortListener, bufferToLowerCasedHeaderName } = require_util();
|
|
28763
28763
|
var { dataURLProcessor, serializeAMimeType, minimizeSupportedMimeType } = require_data_url();
|
|
28764
28764
|
var { getGlobalDispatcher } = require_global2();
|
|
@@ -29749,7 +29749,7 @@ var require_fetch = __commonJS({
|
|
|
29749
29749
|
status: status2,
|
|
29750
29750
|
statusText,
|
|
29751
29751
|
headersList,
|
|
29752
|
-
body: decoders.length ?
|
|
29752
|
+
body: decoders.length ? pipeline2(this.body, ...decoders, (err) => {
|
|
29753
29753
|
if (err) {
|
|
29754
29754
|
this.onError(err);
|
|
29755
29755
|
}
|
|
@@ -32222,9 +32222,9 @@ var require_sender = __commonJS({
|
|
|
32222
32222
|
}
|
|
32223
32223
|
async #run() {
|
|
32224
32224
|
this.#running = true;
|
|
32225
|
-
const
|
|
32226
|
-
while (!
|
|
32227
|
-
const node =
|
|
32225
|
+
const queue2 = this.#queue;
|
|
32226
|
+
while (!queue2.isEmpty()) {
|
|
32227
|
+
const node = queue2.shift();
|
|
32228
32228
|
if (node.promise !== null) {
|
|
32229
32229
|
await node.promise;
|
|
32230
32230
|
}
|
|
@@ -33383,7 +33383,7 @@ ${value}`;
|
|
|
33383
33383
|
var require_eventsource = __commonJS({
|
|
33384
33384
|
"../../node_modules/.pnpm/undici@7.25.0/node_modules/undici/lib/web/eventsource/eventsource.js"(exports, module) {
|
|
33385
33385
|
"use strict";
|
|
33386
|
-
var { pipeline } = __require("stream");
|
|
33386
|
+
var { pipeline: pipeline2 } = __require("stream");
|
|
33387
33387
|
var { fetching } = require_fetch();
|
|
33388
33388
|
var { makeRequest } = require_request2();
|
|
33389
33389
|
var { webidl } = require_webidl();
|
|
@@ -33541,7 +33541,7 @@ var require_eventsource = __commonJS({
|
|
|
33541
33541
|
));
|
|
33542
33542
|
}
|
|
33543
33543
|
});
|
|
33544
|
-
|
|
33544
|
+
pipeline2(
|
|
33545
33545
|
response.body.stream,
|
|
33546
33546
|
eventSourceStream,
|
|
33547
33547
|
(error) => {
|
|
@@ -33896,11 +33896,13 @@ function expBackoff(attempt) {
|
|
|
33896
33896
|
const i = Math.min(Math.max(attempt, 0), BACKOFF_MS.length - 1);
|
|
33897
33897
|
return BACKOFF_MS[i];
|
|
33898
33898
|
}
|
|
33899
|
-
var INGEST_BATCH_MAX_EVENTS, BACKOFF_MS, BACKSTOP_SCAN_MS;
|
|
33899
|
+
var INGEST_BATCH_MAX_EVENTS, SESSION_DEBOUNCE_MS, FORCE_SHIP_THRESHOLD, BACKOFF_MS, BACKSTOP_SCAN_MS;
|
|
33900
33900
|
var init_config = __esm({
|
|
33901
33901
|
"../../packages/daemon-core/src/config/index.ts"() {
|
|
33902
33902
|
"use strict";
|
|
33903
33903
|
INGEST_BATCH_MAX_EVENTS = 1e3;
|
|
33904
|
+
SESSION_DEBOUNCE_MS = 7e3;
|
|
33905
|
+
FORCE_SHIP_THRESHOLD = 200;
|
|
33904
33906
|
BACKOFF_MS = [1e3, 2500, 5e3, 1e4, 2e4, 6e4];
|
|
33905
33907
|
BACKSTOP_SCAN_MS = 5 * 6e4;
|
|
33906
33908
|
}
|
|
@@ -34890,7 +34892,7 @@ async function fetchDeviceViewLedgerByClaim(claimCode) {
|
|
|
34890
34892
|
}
|
|
34891
34893
|
function ingestClient() {
|
|
34892
34894
|
if (_ingest) return _ingest;
|
|
34893
|
-
const
|
|
34895
|
+
const logger2 = createLogger("daemon.ingest");
|
|
34894
34896
|
_ingest = new IngestClient({
|
|
34895
34897
|
apiUrl: state.apiUrl,
|
|
34896
34898
|
auth: {
|
|
@@ -34906,7 +34908,7 @@ function ingestClient() {
|
|
|
34906
34908
|
}
|
|
34907
34909
|
}
|
|
34908
34910
|
},
|
|
34909
|
-
logger
|
|
34911
|
+
logger: logger2
|
|
34910
34912
|
});
|
|
34911
34913
|
return _ingest;
|
|
34912
34914
|
}
|
|
@@ -35087,17 +35089,101 @@ var init_ids2 = __esm({
|
|
|
35087
35089
|
});
|
|
35088
35090
|
|
|
35089
35091
|
// ../../packages/daemon-core/src/queue/index.ts
|
|
35092
|
+
async function buildBatches(opts) {
|
|
35093
|
+
const debounceMs = opts.debounceMs ?? SESSION_DEBOUNCE_MS;
|
|
35094
|
+
const maxEvents = opts.maxEvents ?? INGEST_BATCH_MAX_EVENTS;
|
|
35095
|
+
const forceThreshold = opts.forceShipThreshold ?? FORCE_SHIP_THRESHOLD;
|
|
35096
|
+
const bySession = await opts.store.listUnsentBySession();
|
|
35097
|
+
const batches = [];
|
|
35098
|
+
let eventsInCurrentBatch = [];
|
|
35099
|
+
let sessionIdsInCurrentBatch = [];
|
|
35100
|
+
let toolCallsInCurrentBatch = [];
|
|
35101
|
+
for (const [, items] of bySession) {
|
|
35102
|
+
if (items.length === 0) continue;
|
|
35103
|
+
const lastTs = Math.max(...items.map((i) => i.last_event_ts_ms));
|
|
35104
|
+
const quiet = opts.nowMs - lastTs >= debounceMs;
|
|
35105
|
+
if (!quiet && items.length < forceThreshold) continue;
|
|
35106
|
+
for (const item of items) {
|
|
35107
|
+
const itemCalls = item.tool_calls ?? [];
|
|
35108
|
+
if (eventsInCurrentBatch.length >= maxEvents || eventsInCurrentBatch.length > 0 && toolCallsInCurrentBatch.length + itemCalls.length > INGEST_BATCH_MAX_TOOL_CALLS) {
|
|
35109
|
+
batches.push(
|
|
35110
|
+
await finalise({
|
|
35111
|
+
events: eventsInCurrentBatch,
|
|
35112
|
+
sessionIds: sessionIdsInCurrentBatch,
|
|
35113
|
+
toolCalls: toolCallsInCurrentBatch,
|
|
35114
|
+
pipeline: opts.pipeline,
|
|
35115
|
+
deviceId: opts.deviceId,
|
|
35116
|
+
daemonVersion: opts.daemonVersion
|
|
35117
|
+
})
|
|
35118
|
+
);
|
|
35119
|
+
eventsInCurrentBatch = [];
|
|
35120
|
+
sessionIdsInCurrentBatch = [];
|
|
35121
|
+
toolCallsInCurrentBatch = [];
|
|
35122
|
+
}
|
|
35123
|
+
eventsInCurrentBatch.push(item.event);
|
|
35124
|
+
sessionIdsInCurrentBatch.push(item.session_id);
|
|
35125
|
+
toolCallsInCurrentBatch.push(...itemCalls);
|
|
35126
|
+
}
|
|
35127
|
+
}
|
|
35128
|
+
if (eventsInCurrentBatch.length > 0) {
|
|
35129
|
+
batches.push(
|
|
35130
|
+
await finalise({
|
|
35131
|
+
events: eventsInCurrentBatch,
|
|
35132
|
+
sessionIds: sessionIdsInCurrentBatch,
|
|
35133
|
+
toolCalls: toolCallsInCurrentBatch,
|
|
35134
|
+
pipeline: opts.pipeline,
|
|
35135
|
+
deviceId: opts.deviceId,
|
|
35136
|
+
daemonVersion: opts.daemonVersion
|
|
35137
|
+
})
|
|
35138
|
+
);
|
|
35139
|
+
}
|
|
35140
|
+
return batches;
|
|
35141
|
+
}
|
|
35142
|
+
function attachSegmentIds(calls, segments) {
|
|
35143
|
+
const segmentByEvent = /* @__PURE__ */ new Map();
|
|
35144
|
+
for (const seg of segments) {
|
|
35145
|
+
for (const id of seg.source_event_ids) segmentByEvent.set(id, seg.segment_id);
|
|
35146
|
+
}
|
|
35147
|
+
return attachSegmentIdsByMap(calls, segmentByEvent);
|
|
35148
|
+
}
|
|
35090
35149
|
function attachSegmentIdsByMap(calls, segmentByEvent) {
|
|
35091
35150
|
return calls.map((c) => ({
|
|
35092
35151
|
...c,
|
|
35093
35152
|
segment_id: segmentByEvent.get(c.source_event_id) ?? null
|
|
35094
35153
|
}));
|
|
35095
35154
|
}
|
|
35155
|
+
async function finalise(args) {
|
|
35156
|
+
const segments = [];
|
|
35157
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
35158
|
+
for (let i = 0; i < args.events.length; i++) {
|
|
35159
|
+
const sid = args.sessionIds[i] ?? args.events[i]?.session_id ?? "unknown";
|
|
35160
|
+
const arr = bySession.get(sid) ?? [];
|
|
35161
|
+
const ev = args.events[i];
|
|
35162
|
+
if (ev) arr.push(ev);
|
|
35163
|
+
bySession.set(sid, arr);
|
|
35164
|
+
}
|
|
35165
|
+
for (const [, sessionEvents] of bySession) {
|
|
35166
|
+
const segs = await args.pipeline.run(sessionEvents);
|
|
35167
|
+
segments.push(...segs);
|
|
35168
|
+
}
|
|
35169
|
+
return {
|
|
35170
|
+
batch_id: batchId(),
|
|
35171
|
+
device_id: args.deviceId,
|
|
35172
|
+
daemon_version: args.daemonVersion,
|
|
35173
|
+
events: args.events,
|
|
35174
|
+
segments,
|
|
35175
|
+
// Segments are in hand here, so segment attribution happens at the
|
|
35176
|
+
// last responsible moment before the wire.
|
|
35177
|
+
tool_calls: attachSegmentIds(args.toolCalls, segments)
|
|
35178
|
+
};
|
|
35179
|
+
}
|
|
35180
|
+
var INGEST_BATCH_MAX_TOOL_CALLS;
|
|
35096
35181
|
var init_queue = __esm({
|
|
35097
35182
|
"../../packages/daemon-core/src/queue/index.ts"() {
|
|
35098
35183
|
"use strict";
|
|
35099
35184
|
init_config();
|
|
35100
35185
|
init_ids2();
|
|
35186
|
+
INGEST_BATCH_MAX_TOOL_CALLS = 2e4;
|
|
35101
35187
|
}
|
|
35102
35188
|
});
|
|
35103
35189
|
|
|
@@ -37090,7 +37176,7 @@ var init_scan = __esm({
|
|
|
37090
37176
|
init_api();
|
|
37091
37177
|
init_config2();
|
|
37092
37178
|
init_pipeline2();
|
|
37093
|
-
DAEMON_VERSION = true ? "daemon-0.
|
|
37179
|
+
DAEMON_VERSION = true ? "daemon-0.8.0" : "daemon-dev";
|
|
37094
37180
|
BATCH_MAX_EVENTS = INGEST_BATCH_MAX_EVENTS;
|
|
37095
37181
|
BATCH_MAX_TOOL_CALLS = 2e4;
|
|
37096
37182
|
BATCH_BUFFER_HARD_CAP = BATCH_MAX_EVENTS * 2;
|
|
@@ -37327,6 +37413,165 @@ var init_update = __esm({
|
|
|
37327
37413
|
}
|
|
37328
37414
|
});
|
|
37329
37415
|
|
|
37416
|
+
// src/receiver.ts
|
|
37417
|
+
import { createServer } from "http";
|
|
37418
|
+
function queue() {
|
|
37419
|
+
if (!store) store = new FileQueueStore(homePath("sdk-ingest-queue.json"));
|
|
37420
|
+
return store;
|
|
37421
|
+
}
|
|
37422
|
+
function parseBatch(json) {
|
|
37423
|
+
if (typeof json !== "object" || json === null) return { error: "body must be a JSON object" };
|
|
37424
|
+
const b = json;
|
|
37425
|
+
if (!Array.isArray(b.events) || b.events.length === 0)
|
|
37426
|
+
return { error: "events must be a non-empty array" };
|
|
37427
|
+
if (b.events.length > 1e4) return { error: "too many events (max 10000)" };
|
|
37428
|
+
for (const e of b.events) {
|
|
37429
|
+
if (typeof e !== "object" || e === null) return { error: "each event must be an object" };
|
|
37430
|
+
const ev = e;
|
|
37431
|
+
for (const k of ["source_event_id", "session_id", "agent", "ts"]) {
|
|
37432
|
+
if (typeof ev[k] !== "string" || ev[k].length === 0)
|
|
37433
|
+
return { error: `event.${k} is required` };
|
|
37434
|
+
}
|
|
37435
|
+
}
|
|
37436
|
+
const toolCalls = Array.isArray(b.tool_calls) ? b.tool_calls : [];
|
|
37437
|
+
return { events: b.events, toolCalls };
|
|
37438
|
+
}
|
|
37439
|
+
async function enqueue(batch) {
|
|
37440
|
+
const q = queue();
|
|
37441
|
+
for (const event of batch.events) {
|
|
37442
|
+
const calls = batch.toolCalls.filter((tc) => tc.source_event_id === event.source_event_id).map(({ segment_id: _segmentId, ...draft }) => draft);
|
|
37443
|
+
await q.put({
|
|
37444
|
+
source_event_id: event.source_event_id,
|
|
37445
|
+
session_id: event.session_id,
|
|
37446
|
+
agent: event.agent,
|
|
37447
|
+
event,
|
|
37448
|
+
last_event_ts_ms: Date.parse(event.ts) || Date.now(),
|
|
37449
|
+
synced: false,
|
|
37450
|
+
sent_batch_id: null,
|
|
37451
|
+
tool_calls: calls.length > 0 ? calls : void 0
|
|
37452
|
+
});
|
|
37453
|
+
}
|
|
37454
|
+
return batch.events.length;
|
|
37455
|
+
}
|
|
37456
|
+
async function drainLocalQueue(opts) {
|
|
37457
|
+
if (draining) return { batches: 0, events: 0 };
|
|
37458
|
+
draining = true;
|
|
37459
|
+
try {
|
|
37460
|
+
const q = queue();
|
|
37461
|
+
if (await q.countUnsent() === 0) return { batches: 0, events: 0 };
|
|
37462
|
+
const batches = await buildBatches({
|
|
37463
|
+
store: q,
|
|
37464
|
+
pipeline,
|
|
37465
|
+
deviceId: opts.deviceId,
|
|
37466
|
+
daemonVersion: opts.daemonVersion,
|
|
37467
|
+
nowMs: Date.now()
|
|
37468
|
+
});
|
|
37469
|
+
let events = 0;
|
|
37470
|
+
for (const batch of batches) {
|
|
37471
|
+
const shipped = {
|
|
37472
|
+
...batch,
|
|
37473
|
+
events: batch.events.map(({ content_excerpt: _excerpt, ...rest }) => rest)
|
|
37474
|
+
};
|
|
37475
|
+
await uploadBatch(shipped);
|
|
37476
|
+
await q.markSent(
|
|
37477
|
+
batch.events.map((e) => e.source_event_id),
|
|
37478
|
+
batch.batch_id
|
|
37479
|
+
);
|
|
37480
|
+
events += batch.events.length;
|
|
37481
|
+
}
|
|
37482
|
+
return { batches: batches.length, events };
|
|
37483
|
+
} finally {
|
|
37484
|
+
draining = false;
|
|
37485
|
+
}
|
|
37486
|
+
}
|
|
37487
|
+
function localQueueDepth() {
|
|
37488
|
+
return queue().countUnsent();
|
|
37489
|
+
}
|
|
37490
|
+
function startLocalIngestReceiver(opts = {}) {
|
|
37491
|
+
const port = opts.port ?? (Number(process.env.MODELSTAT_LOCAL_INGEST_PORT) || DEFAULT_LOCAL_INGEST_PORT);
|
|
37492
|
+
return new Promise((resolve6) => {
|
|
37493
|
+
const server = createServer((req, res) => void handle(req, res));
|
|
37494
|
+
let settled = false;
|
|
37495
|
+
server.on("error", (err) => {
|
|
37496
|
+
if (settled) return;
|
|
37497
|
+
settled = true;
|
|
37498
|
+
logger.warn(
|
|
37499
|
+
`local ingest receiver disabled \u2014 SDK local_daemon mode unavailable: ${err.code ?? err.message} on 127.0.0.1:${port}`
|
|
37500
|
+
);
|
|
37501
|
+
resolve6(null);
|
|
37502
|
+
});
|
|
37503
|
+
server.listen(port, "127.0.0.1", () => {
|
|
37504
|
+
settled = true;
|
|
37505
|
+
const addr = server.address();
|
|
37506
|
+
const boundPort = typeof addr === "object" && addr ? addr.port : port;
|
|
37507
|
+
logger.info(`local ingest receiver on http://127.0.0.1:${boundPort}/v1/ingest`);
|
|
37508
|
+
resolve6({
|
|
37509
|
+
port: boundPort,
|
|
37510
|
+
close: () => new Promise((r) => server.close(() => r()))
|
|
37511
|
+
});
|
|
37512
|
+
});
|
|
37513
|
+
});
|
|
37514
|
+
}
|
|
37515
|
+
async function handle(req, res) {
|
|
37516
|
+
const send = (code, body) => {
|
|
37517
|
+
res.writeHead(code, { "content-type": "application/json" });
|
|
37518
|
+
res.end(JSON.stringify(body));
|
|
37519
|
+
};
|
|
37520
|
+
const path = (req.url ?? "").split("?")[0];
|
|
37521
|
+
if (req.method === "GET" && path === "/healthz") return send(200, { ok: true });
|
|
37522
|
+
if (req.method !== "POST" || path !== "/v1/ingest" && path !== "/v1/ingest/raw") {
|
|
37523
|
+
return send(404, { error: "not found" });
|
|
37524
|
+
}
|
|
37525
|
+
let size = 0;
|
|
37526
|
+
const chunks = [];
|
|
37527
|
+
try {
|
|
37528
|
+
for await (const chunk of req) {
|
|
37529
|
+
size += chunk.length;
|
|
37530
|
+
if (size > MAX_BODY_BYTES) {
|
|
37531
|
+
send(413, { error: "batch too large" });
|
|
37532
|
+
req.destroy();
|
|
37533
|
+
return;
|
|
37534
|
+
}
|
|
37535
|
+
chunks.push(chunk);
|
|
37536
|
+
}
|
|
37537
|
+
} catch {
|
|
37538
|
+
return send(400, { error: "read error" });
|
|
37539
|
+
}
|
|
37540
|
+
let json;
|
|
37541
|
+
try {
|
|
37542
|
+
json = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
37543
|
+
} catch {
|
|
37544
|
+
return send(400, { error: "invalid json" });
|
|
37545
|
+
}
|
|
37546
|
+
const parsed = parseBatch(json);
|
|
37547
|
+
if ("error" in parsed) return send(400, parsed);
|
|
37548
|
+
try {
|
|
37549
|
+
const accepted = await enqueue(parsed);
|
|
37550
|
+
return send(200, { accepted, queued: true });
|
|
37551
|
+
} catch (e) {
|
|
37552
|
+
logger.warn(`local ingest enqueue failed: ${e.message}`);
|
|
37553
|
+
return send(500, { error: "enqueue failed" });
|
|
37554
|
+
}
|
|
37555
|
+
}
|
|
37556
|
+
var logger, DEFAULT_LOCAL_INGEST_PORT, MAX_BODY_BYTES, store, pipeline, draining;
|
|
37557
|
+
var init_receiver = __esm({
|
|
37558
|
+
"src/receiver.ts"() {
|
|
37559
|
+
"use strict";
|
|
37560
|
+
init_logger();
|
|
37561
|
+
init_node();
|
|
37562
|
+
init_queue();
|
|
37563
|
+
init_api();
|
|
37564
|
+
init_paths();
|
|
37565
|
+
init_pipeline2();
|
|
37566
|
+
logger = createLogger("daemon.local-ingest");
|
|
37567
|
+
DEFAULT_LOCAL_INGEST_PORT = 4319;
|
|
37568
|
+
MAX_BODY_BYTES = 16 * 1024 * 1024;
|
|
37569
|
+
store = null;
|
|
37570
|
+
pipeline = { run: (events) => buildSegments(events) };
|
|
37571
|
+
draining = false;
|
|
37572
|
+
}
|
|
37573
|
+
});
|
|
37574
|
+
|
|
37330
37575
|
// src/reconcile.ts
|
|
37331
37576
|
import { stat as stat3 } from "fs/promises";
|
|
37332
37577
|
function utcDay(ts) {
|
|
@@ -37704,19 +37949,19 @@ function createPolicyRefresher(opts) {
|
|
|
37704
37949
|
...opts.fetch ? { fetch: opts.fetch } : {},
|
|
37705
37950
|
...opts.logger ? { logger: opts.logger } : {}
|
|
37706
37951
|
};
|
|
37707
|
-
const
|
|
37952
|
+
const store2 = new RemoteConfigStore(env, [policiesKind]);
|
|
37708
37953
|
let timer = null;
|
|
37709
37954
|
const apply = () => {
|
|
37710
|
-
const bundle =
|
|
37955
|
+
const bundle = store2.get("policies");
|
|
37711
37956
|
setRemoteRedactionPatterns(compilePolicyPatterns(bundle));
|
|
37712
37957
|
};
|
|
37713
37958
|
const refresh = async () => {
|
|
37714
|
-
await
|
|
37959
|
+
await store2.refresh("policies");
|
|
37715
37960
|
apply();
|
|
37716
37961
|
};
|
|
37717
37962
|
return {
|
|
37718
37963
|
async start() {
|
|
37719
|
-
await
|
|
37964
|
+
await store2.initFromCache();
|
|
37720
37965
|
apply();
|
|
37721
37966
|
void refresh().catch(() => {
|
|
37722
37967
|
});
|
|
@@ -39743,6 +39988,25 @@ async function runDaemon(opts = {}) {
|
|
|
39743
39988
|
setPhase("error", `summariser preflight failed: ${err.message}`);
|
|
39744
39989
|
throw err;
|
|
39745
39990
|
}
|
|
39991
|
+
const localIngest = await startLocalIngestReceiver();
|
|
39992
|
+
const LOCAL_DRAIN_INTERVAL_MS = 5e3;
|
|
39993
|
+
let localDrainTimer = null;
|
|
39994
|
+
if (localIngest) {
|
|
39995
|
+
const drainTick = async () => {
|
|
39996
|
+
try {
|
|
39997
|
+
const { events } = await drainLocalQueue({
|
|
39998
|
+
deviceId: state.deviceId,
|
|
39999
|
+
daemonVersion: DAEMON_VERSION2
|
|
40000
|
+
});
|
|
40001
|
+
if (events > 0) bumpStat("sdk_events_uploaded", events);
|
|
40002
|
+
setStat("sdk_queue", await localQueueDepth());
|
|
40003
|
+
} catch (e) {
|
|
40004
|
+
setMessage(`SDK ingest upload deferred: ${describeErrorWithCause(e)}`);
|
|
40005
|
+
}
|
|
40006
|
+
};
|
|
40007
|
+
localDrainTimer = setInterval(() => void drainTick(), LOCAL_DRAIN_INTERVAL_MS);
|
|
40008
|
+
localDrainTimer.unref();
|
|
40009
|
+
}
|
|
39746
40010
|
const { reconcileProcessingVersion: reconcileProcessingVersion2 } = await Promise.resolve().then(() => (init_processing_version(), processing_version_exports));
|
|
39747
40011
|
const pv = reconcileProcessingVersion2(state);
|
|
39748
40012
|
if (pv.changed) {
|
|
@@ -39809,6 +40073,8 @@ async function runDaemon(opts = {}) {
|
|
|
39809
40073
|
setPhase("offline", "Shutting down");
|
|
39810
40074
|
await sendHeartbeat();
|
|
39811
40075
|
await watcher.close();
|
|
40076
|
+
if (localDrainTimer) clearInterval(localDrainTimer);
|
|
40077
|
+
await localIngest?.close();
|
|
39812
40078
|
try {
|
|
39813
40079
|
const { disposeLlama: disposeLlama2 } = await Promise.resolve().then(() => (init_node(), node_exports));
|
|
39814
40080
|
await disposeLlama2();
|
|
@@ -39832,11 +40098,12 @@ var init_daemon = __esm({
|
|
|
39832
40098
|
init_config2();
|
|
39833
40099
|
init_lock();
|
|
39834
40100
|
init_machine_key();
|
|
40101
|
+
init_receiver();
|
|
39835
40102
|
init_reconcile();
|
|
39836
40103
|
init_scan();
|
|
39837
40104
|
init_single_flight();
|
|
39838
40105
|
init_update();
|
|
39839
|
-
DAEMON_VERSION2 = true ? "daemon-0.
|
|
40106
|
+
DAEMON_VERSION2 = true ? "daemon-0.8.0" : "daemon-dev";
|
|
39840
40107
|
HEARTBEAT_INTERVAL_MS = 1e4;
|
|
39841
40108
|
SCAN_INTERVAL_MS = 5 * 60 * 1e3;
|
|
39842
40109
|
DISCOVERY_INTERVAL_MS = 6e4;
|
|
@@ -40440,7 +40707,7 @@ function tryOpenBrowser(url) {
|
|
|
40440
40707
|
return false;
|
|
40441
40708
|
}
|
|
40442
40709
|
}
|
|
40443
|
-
var DAEMON_VERSION3 = true ? "daemon-0.
|
|
40710
|
+
var DAEMON_VERSION3 = true ? "daemon-0.8.0" : "daemon-dev";
|
|
40444
40711
|
function osFamily() {
|
|
40445
40712
|
const p = platform6();
|
|
40446
40713
|
if (p === "darwin") return "macos";
|