@massa-ai/mcp-client 1.37.0 → 1.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-cli.js +1335 -1316
- package/dist/index.js +19 -0
- package/package.json +3 -3
package/dist/config-cli.js
CHANGED
|
@@ -108231,6 +108231,7 @@ var init_registry = __esm(() => {
|
|
|
108231
108231
|
search_analytics: { storeId: "search_analytics", identityColumn: "project_id", mutable: true },
|
|
108232
108232
|
search_events: { storeId: "search_events", identityColumn: "project_id", mutable: true },
|
|
108233
108233
|
synapse_sessions: { storeId: "synapse_sessions", identityColumn: "workspace_id", mutable: true },
|
|
108234
|
+
managed_runs: { storeId: "managed_runs", identityColumn: "project_id", mutable: true },
|
|
108234
108235
|
operation_log: { storeId: "operation_log", identityColumn: "project_id", mutable: false },
|
|
108235
108236
|
project_identity_operations: {
|
|
108236
108237
|
storeId: "project_identity_operations",
|
|
@@ -127386,1409 +127387,1427 @@ var init_observation_consolidation_job = __esm(() => {
|
|
|
127386
127387
|
observationConsolidationJob = new ObservationConsolidationJob;
|
|
127387
127388
|
});
|
|
127388
127389
|
|
|
127389
|
-
// ../../packages/core/dist/services/
|
|
127390
|
-
function
|
|
127391
|
-
|
|
127392
|
-
|
|
127393
|
-
|
|
127394
|
-
return raw2 === "true" || raw2 === "1";
|
|
127395
|
-
}
|
|
127396
|
-
function envNum2(key, fallback) {
|
|
127397
|
-
const raw2 = process.env[key];
|
|
127398
|
-
if (raw2 === undefined || raw2 === "")
|
|
127399
|
-
return fallback;
|
|
127400
|
-
const n = Number(raw2);
|
|
127401
|
-
return Number.isFinite(n) && n > 0 ? n : fallback;
|
|
127390
|
+
// ../../packages/core/dist/services/checkpoint/checkpoint-store-pg.js
|
|
127391
|
+
function toNum4(v) {
|
|
127392
|
+
if (v == null)
|
|
127393
|
+
return null;
|
|
127394
|
+
return typeof v === "bigint" ? Number(v) : v;
|
|
127402
127395
|
}
|
|
127403
|
-
function
|
|
127404
|
-
|
|
127405
|
-
|
|
127406
|
-
}
|
|
127407
|
-
if (job.jobKind === "memory-consolidation") {
|
|
127408
|
-
const currentInterval = job.schedule.intervalMs ?? THIRTY_MIN;
|
|
127409
|
-
const safeInterval = Math.max(currentInterval, THIRTY_MIN);
|
|
127410
|
-
return {
|
|
127411
|
-
...job,
|
|
127412
|
-
defaultEnabled: true,
|
|
127413
|
-
schedule: { type: "interval", intervalMs: safeInterval }
|
|
127414
|
-
};
|
|
127415
|
-
}
|
|
127416
|
-
if (job.jobKind === "decay-sweep") {
|
|
127417
|
-
const currentInterval = job.schedule.intervalMs ?? ONE_HOUR;
|
|
127418
|
-
const safeInterval = Math.max(currentInterval, ONE_HOUR);
|
|
127419
|
-
return {
|
|
127420
|
-
...job,
|
|
127421
|
-
defaultEnabled: true,
|
|
127422
|
-
schedule: { type: "interval", intervalMs: safeInterval }
|
|
127423
|
-
};
|
|
127424
|
-
}
|
|
127425
|
-
return job;
|
|
127396
|
+
function compressState(state) {
|
|
127397
|
+
const json3 = JSON.stringify(state);
|
|
127398
|
+
return Buffer.from(Bun.deflateSync(Buffer.from(json3, "utf-8")));
|
|
127426
127399
|
}
|
|
127427
|
-
function
|
|
127428
|
-
|
|
127429
|
-
|
|
127430
|
-
|
|
127431
|
-
});
|
|
127432
|
-
scheduler.registerHandler("decay-sweep", async () => {
|
|
127433
|
-
const { memoryConsolidationJob: memoryConsolidationJob2 } = await Promise.resolve().then(() => (init_memory_consolidation_job(), exports_memory_consolidation_job));
|
|
127434
|
-
await memoryConsolidationJob2.consolidate();
|
|
127435
|
-
});
|
|
127436
|
-
scheduler.registerHandler("auto-improve", async (job) => {
|
|
127437
|
-
const { autoImproveJob: autoImproveJob2 } = await Promise.resolve().then(() => (init_auto_improve_job(), exports_auto_improve_job));
|
|
127438
|
-
const projectId = job.payload?.projectId ?? "default";
|
|
127439
|
-
await autoImproveJob2.runOnce(projectId);
|
|
127440
|
-
});
|
|
127441
|
-
scheduler.registerHandler("observation-bridge", async (job) => {
|
|
127442
|
-
const { observationConsolidationJob: observationConsolidationJob2 } = await Promise.resolve().then(() => (init_observation_consolidation_job(), exports_observation_consolidation_job));
|
|
127443
|
-
const projectId = job.payload?.projectId ?? "default";
|
|
127444
|
-
await observationConsolidationJob2.runOnce(projectId);
|
|
127445
|
-
});
|
|
127446
|
-
for (const rawDef of DEFAULT_SCHEDULED_JOBS) {
|
|
127447
|
-
const def = applySafeDefaults(rawDef);
|
|
127448
|
-
const enabled = envBool2(def.enableEnvVar, def.defaultEnabled);
|
|
127449
|
-
const intervalMs = envNum2(def.intervalEnvVar, def.schedule.intervalMs ?? THIRTY_MIN);
|
|
127450
|
-
const schedule = { type: "interval", intervalMs };
|
|
127451
|
-
scheduler.registerOrResumeJob({
|
|
127452
|
-
id: def.id,
|
|
127453
|
-
name: def.name,
|
|
127454
|
-
jobKind: def.jobKind,
|
|
127455
|
-
schedule,
|
|
127456
|
-
nextRunAt: 0,
|
|
127457
|
-
enabled
|
|
127458
|
-
});
|
|
127459
|
-
}
|
|
127460
|
-
logger.info("Scheduler default jobs registered", {
|
|
127461
|
-
handlers: scheduler.registeredKinds(),
|
|
127462
|
-
jobs: DEFAULT_SCHEDULED_JOBS.length
|
|
127463
|
-
});
|
|
127400
|
+
function decompressState(data) {
|
|
127401
|
+
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
127402
|
+
const inflated = Bun.inflateSync(new Uint8Array(buf));
|
|
127403
|
+
return JSON.parse(Buffer.from(inflated).toString("utf-8"));
|
|
127464
127404
|
}
|
|
127465
|
-
var MIN, THIRTY_MIN, ONE_HOUR, DEFAULT_SCHEDULED_JOBS;
|
|
127466
|
-
var init_scheduler_defaults = __esm(() => {
|
|
127467
|
-
init_dist();
|
|
127468
|
-
MIN = 60 * 1000;
|
|
127469
|
-
THIRTY_MIN = 30 * MIN;
|
|
127470
|
-
ONE_HOUR = 60 * MIN;
|
|
127471
|
-
DEFAULT_SCHEDULED_JOBS = [
|
|
127472
|
-
{
|
|
127473
|
-
id: "scheduled-memory-consolidation",
|
|
127474
|
-
name: "Memory Consolidation (clock)",
|
|
127475
|
-
jobKind: "memory-consolidation",
|
|
127476
|
-
schedule: { type: "interval", intervalMs: THIRTY_MIN },
|
|
127477
|
-
defaultEnabled: false,
|
|
127478
|
-
enableEnvVar: "MASSA_AI_SCHEDULER_CONSOLIDATION_ENABLED",
|
|
127479
|
-
intervalEnvVar: "MASSA_AI_SCHEDULER_CONSOLIDATION_INTERVAL_MS"
|
|
127480
|
-
},
|
|
127481
|
-
{
|
|
127482
|
-
id: "scheduled-decay-sweep",
|
|
127483
|
-
name: "Decay Sweep (clock)",
|
|
127484
|
-
jobKind: "decay-sweep",
|
|
127485
|
-
schedule: { type: "interval", intervalMs: ONE_HOUR },
|
|
127486
|
-
defaultEnabled: false,
|
|
127487
|
-
enableEnvVar: "MASSA_AI_SCHEDULER_DECAY_ENABLED",
|
|
127488
|
-
intervalEnvVar: "MASSA_AI_SCHEDULER_DECAY_INTERVAL_MS"
|
|
127489
|
-
},
|
|
127490
|
-
{
|
|
127491
|
-
id: "scheduled-auto-improve",
|
|
127492
|
-
name: "Auto-Improve (clock)",
|
|
127493
|
-
jobKind: "auto-improve",
|
|
127494
|
-
schedule: { type: "interval", intervalMs: THIRTY_MIN },
|
|
127495
|
-
defaultEnabled: false,
|
|
127496
|
-
enableEnvVar: "MASSA_AI_SCHEDULER_AUTO_IMPROVE_ENABLED",
|
|
127497
|
-
intervalEnvVar: "MASSA_AI_SCHEDULER_AUTO_IMPROVE_INTERVAL_MS"
|
|
127498
|
-
},
|
|
127499
|
-
{
|
|
127500
|
-
id: "scheduled-observation-bridge",
|
|
127501
|
-
name: "Observation Bridge (clock)",
|
|
127502
|
-
jobKind: "observation-bridge",
|
|
127503
|
-
schedule: { type: "interval", intervalMs: THIRTY_MIN },
|
|
127504
|
-
defaultEnabled: false,
|
|
127505
|
-
enableEnvVar: "MASSA_AI_SCHEDULER_OBSERVATION_BRIDGE_ENABLED",
|
|
127506
|
-
intervalEnvVar: "MASSA_AI_SCHEDULER_OBSERVATION_BRIDGE_INTERVAL_MS"
|
|
127507
|
-
}
|
|
127508
|
-
];
|
|
127509
|
-
});
|
|
127510
|
-
|
|
127511
|
-
// ../../packages/core/dist/services/scheduler/index.js
|
|
127512
|
-
var init_scheduler2 = __esm(() => {
|
|
127513
|
-
init_scheduler();
|
|
127514
|
-
init_scheduler_store_pg();
|
|
127515
|
-
init_scheduler_store_factory();
|
|
127516
|
-
init_scheduler_cron();
|
|
127517
|
-
init_scheduler_defaults();
|
|
127518
|
-
});
|
|
127519
127405
|
|
|
127520
|
-
|
|
127521
|
-
|
|
127522
|
-
|
|
127523
|
-
|
|
127524
|
-
|
|
127525
|
-
|
|
127526
|
-
|
|
127406
|
+
class PgCheckpointStore {
|
|
127407
|
+
prisma;
|
|
127408
|
+
mirror = new Map;
|
|
127409
|
+
hydrated = false;
|
|
127410
|
+
hydrating = null;
|
|
127411
|
+
hydrateFailedAt = 0;
|
|
127412
|
+
static HYDRATE_RETRY_MS = 30000;
|
|
127413
|
+
inflight = new Map;
|
|
127414
|
+
getClient() {
|
|
127415
|
+
if (!this.prisma)
|
|
127416
|
+
this.prisma = getPrismaClient2();
|
|
127417
|
+
return this.prisma;
|
|
127527
127418
|
}
|
|
127528
|
-
|
|
127529
|
-
|
|
127530
|
-
|
|
127531
|
-
|
|
127532
|
-
|
|
127533
|
-
|
|
127534
|
-
|
|
127535
|
-
static API_URL = "https://models.dev/api.json";
|
|
127536
|
-
static MEMORY_CACHE_TTL = 3600 * 1000;
|
|
127537
|
-
static LOCAL_CACHE_TTL = 24 * 3600 * 1000;
|
|
127538
|
-
static LOCAL_CACHE_FILE = "pricing-cache.json";
|
|
127539
|
-
memoryCache = null;
|
|
127540
|
-
memoryCacheTimestamp = 0;
|
|
127541
|
-
getLocalCachePath() {
|
|
127542
|
-
const dataDir = config.get("dataDir");
|
|
127543
|
-
return path16.join(dataDir, ModelsDevClient.LOCAL_CACHE_FILE);
|
|
127419
|
+
ensureHydrated() {
|
|
127420
|
+
if (this.hydrated)
|
|
127421
|
+
return Promise.resolve();
|
|
127422
|
+
if (this.hydrating)
|
|
127423
|
+
return this.hydrating;
|
|
127424
|
+
if (this.hydrateFailedAt > 0 && Date.now() - this.hydrateFailedAt < PgCheckpointStore.HYDRATE_RETRY_MS) {
|
|
127425
|
+
return Promise.resolve();
|
|
127544
127426
|
}
|
|
127545
|
-
async
|
|
127546
|
-
const cachePath = this.getLocalCachePath();
|
|
127427
|
+
this.hydrating = (async () => {
|
|
127547
127428
|
try {
|
|
127548
|
-
|
|
127549
|
-
|
|
127550
|
-
|
|
127551
|
-
|
|
127552
|
-
const
|
|
127553
|
-
const
|
|
127554
|
-
|
|
127555
|
-
|
|
127556
|
-
|
|
127557
|
-
});
|
|
127429
|
+
const prisma2 = this.getClient();
|
|
127430
|
+
const rows = await prisma2.$queryRaw`
|
|
127431
|
+
SELECT * FROM task_checkpoints
|
|
127432
|
+
`;
|
|
127433
|
+
const next = new Map;
|
|
127434
|
+
const dbIds = new Set;
|
|
127435
|
+
for (const row of rows) {
|
|
127436
|
+
dbIds.add(row.id);
|
|
127437
|
+
next.set(row.id, this.rowToCheckpoint(row));
|
|
127558
127438
|
}
|
|
127559
|
-
const
|
|
127560
|
-
|
|
127561
|
-
|
|
127439
|
+
for (const [id, existing] of this.mirror) {
|
|
127440
|
+
if (!dbIds.has(id))
|
|
127441
|
+
next.set(id, existing);
|
|
127562
127442
|
}
|
|
127563
|
-
|
|
127564
|
-
|
|
127565
|
-
|
|
127443
|
+
this.mirror = next;
|
|
127444
|
+
this.hydrated = true;
|
|
127445
|
+
this.hydrateFailedAt = 0;
|
|
127446
|
+
logger.info("PgCheckpointStore hydrated", {
|
|
127447
|
+
rows: this.mirror.size
|
|
127566
127448
|
});
|
|
127567
|
-
|
|
127568
|
-
|
|
127569
|
-
logger.
|
|
127570
|
-
error:
|
|
127449
|
+
} catch (e) {
|
|
127450
|
+
this.hydrateFailedAt = Date.now();
|
|
127451
|
+
logger.warn("PgCheckpointStore hydrate failed (best-effort)", {
|
|
127452
|
+
error: e.message
|
|
127571
127453
|
});
|
|
127572
|
-
|
|
127454
|
+
} finally {
|
|
127455
|
+
this.hydrating = null;
|
|
127573
127456
|
}
|
|
127457
|
+
})();
|
|
127458
|
+
return this.hydrating;
|
|
127459
|
+
}
|
|
127460
|
+
rowToCheckpoint(row) {
|
|
127461
|
+
const state = decompressState(row.state);
|
|
127462
|
+
const storedSchemaVersion = toNum4(row.state_schema_version);
|
|
127463
|
+
if (storedSchemaVersion != null) {
|
|
127464
|
+
const storedVersionString = String(storedSchemaVersion);
|
|
127465
|
+
const normalized = /^\d+$/u.test(storedVersionString) ? `${storedVersionString}.0.0` : storedVersionString;
|
|
127466
|
+
assertSchemaSupported("checkpoint", normalized, SUPPORTED_CHECKPOINT_STATE_SCHEMA_VERSION);
|
|
127574
127467
|
}
|
|
127575
|
-
|
|
127576
|
-
|
|
127577
|
-
|
|
127578
|
-
|
|
127579
|
-
|
|
127580
|
-
|
|
127581
|
-
|
|
127582
|
-
|
|
127583
|
-
|
|
127584
|
-
|
|
127585
|
-
|
|
127586
|
-
|
|
127587
|
-
|
|
127588
|
-
|
|
127589
|
-
|
|
127590
|
-
|
|
127591
|
-
|
|
127592
|
-
|
|
127593
|
-
|
|
127594
|
-
|
|
127595
|
-
|
|
127596
|
-
|
|
127597
|
-
|
|
127598
|
-
|
|
127599
|
-
|
|
127600
|
-
|
|
127601
|
-
|
|
127602
|
-
|
|
127603
|
-
|
|
127604
|
-
|
|
127605
|
-
|
|
127606
|
-
|
|
127607
|
-
|
|
127608
|
-
|
|
127609
|
-
|
|
127610
|
-
|
|
127611
|
-
|
|
127612
|
-
|
|
127613
|
-
|
|
127614
|
-
|
|
127615
|
-
|
|
127616
|
-
|
|
127617
|
-
|
|
127618
|
-
|
|
127619
|
-
|
|
127620
|
-
|
|
127621
|
-
|
|
127622
|
-
|
|
127623
|
-
|
|
127624
|
-
|
|
127625
|
-
|
|
127626
|
-
|
|
127627
|
-
|
|
127628
|
-
|
|
127629
|
-
|
|
127630
|
-
|
|
127631
|
-
|
|
127632
|
-
|
|
127633
|
-
|
|
127634
|
-
|
|
127635
|
-
|
|
127636
|
-
|
|
127637
|
-
|
|
127638
|
-
|
|
127639
|
-
|
|
127640
|
-
|
|
127641
|
-
|
|
127642
|
-
|
|
127643
|
-
|
|
127644
|
-
|
|
127645
|
-
|
|
127646
|
-
|
|
127647
|
-
|
|
127648
|
-
|
|
127649
|
-
|
|
127650
|
-
|
|
127651
|
-
|
|
127652
|
-
|
|
127653
|
-
|
|
127654
|
-
|
|
127655
|
-
|
|
127656
|
-
|
|
127657
|
-
|
|
127658
|
-
|
|
127659
|
-
|
|
127660
|
-
|
|
127661
|
-
|
|
127662
|
-
|
|
127663
|
-
|
|
127664
|
-
|
|
127665
|
-
|
|
127666
|
-
|
|
127667
|
-
|
|
127668
|
-
|
|
127669
|
-
|
|
127670
|
-
|
|
127671
|
-
|
|
127672
|
-
|
|
127673
|
-
|
|
127674
|
-
|
|
127675
|
-
|
|
127676
|
-
|
|
127677
|
-
|
|
127678
|
-
|
|
127679
|
-
|
|
127680
|
-
|
|
127681
|
-
|
|
127682
|
-
|
|
127683
|
-
|
|
127684
|
-
|
|
127685
|
-
|
|
127686
|
-
|
|
127687
|
-
|
|
127688
|
-
|
|
127689
|
-
|
|
127690
|
-
|
|
127691
|
-
|
|
127692
|
-
|
|
127693
|
-
|
|
127694
|
-
id: "gemini-1.5-flash",
|
|
127695
|
-
name: "Gemini 1.5 Flash",
|
|
127696
|
-
provider: "Google",
|
|
127697
|
-
family: "gemini",
|
|
127698
|
-
inputCostPerMillion: 0.075,
|
|
127699
|
-
outputCostPerMillion: 0.3,
|
|
127700
|
-
contextWindow: 1e6,
|
|
127701
|
-
maxOutputTokens: 8192
|
|
127702
|
-
},
|
|
127703
|
-
{
|
|
127704
|
-
id: "embed-english-v3.0",
|
|
127705
|
-
name: "Embed English v3",
|
|
127706
|
-
provider: "Cohere",
|
|
127707
|
-
family: "embed",
|
|
127708
|
-
inputCostPerMillion: 0.1,
|
|
127709
|
-
outputCostPerMillion: 0,
|
|
127710
|
-
contextWindow: 512
|
|
127711
|
-
}
|
|
127712
|
-
];
|
|
127713
|
-
for (const pricing of defaults3) {
|
|
127714
|
-
models.set(pricing.id, pricing);
|
|
127715
|
-
}
|
|
127716
|
-
logger.info("Using bundled pricing defaults (offline mode)", {
|
|
127717
|
-
models: models.size
|
|
127718
|
-
});
|
|
127719
|
-
return models;
|
|
127720
|
-
}
|
|
127721
|
-
async fetchAllModels() {
|
|
127722
|
-
if (this.memoryCache && Date.now() - this.memoryCacheTimestamp < ModelsDevClient.MEMORY_CACHE_TTL) {
|
|
127723
|
-
logger.debug("Using in-memory pricing cache");
|
|
127724
|
-
return this.memoryCache;
|
|
127725
|
-
}
|
|
127726
|
-
const localCache = await this.loadLocalCache();
|
|
127727
|
-
let remoteModels = null;
|
|
127728
|
-
try {
|
|
127729
|
-
remoteModels = await this.fetchRemote();
|
|
127730
|
-
await this.saveLocalCache(remoteModels);
|
|
127731
|
-
} catch (error51) {
|
|
127732
|
-
logger.debug("Remote pricing fetch failed (offline?)", {
|
|
127733
|
-
error: error51.message
|
|
127734
|
-
});
|
|
127735
|
-
}
|
|
127736
|
-
let result;
|
|
127737
|
-
if (remoteModels) {
|
|
127738
|
-
result = remoteModels;
|
|
127739
|
-
logger.debug("Using remote pricing data");
|
|
127740
|
-
} else if (localCache) {
|
|
127741
|
-
result = localCache;
|
|
127742
|
-
logger.info("Using local pricing cache (offline mode)");
|
|
127743
|
-
} else {
|
|
127744
|
-
result = this.getBundledDefaults();
|
|
127745
|
-
}
|
|
127746
|
-
this.memoryCache = result;
|
|
127747
|
-
this.memoryCacheTimestamp = Date.now();
|
|
127748
|
-
return result;
|
|
127749
|
-
}
|
|
127750
|
-
async fetchRemote() {
|
|
127751
|
-
const controller = new AbortController;
|
|
127752
|
-
const timeout = setTimeout(() => controller.abort(), 1e4);
|
|
127753
|
-
try {
|
|
127754
|
-
logger.info("Fetching pricing data from models.dev API");
|
|
127755
|
-
const response = await fetch(ModelsDevClient.API_URL, {
|
|
127756
|
-
signal: controller.signal
|
|
127757
|
-
});
|
|
127758
|
-
if (!response.ok) {
|
|
127759
|
-
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
127760
|
-
}
|
|
127761
|
-
const data = await response.json();
|
|
127762
|
-
const models = new Map;
|
|
127763
|
-
let totalModels = 0;
|
|
127764
|
-
let modelsWithPricing = 0;
|
|
127765
|
-
for (const [providerId, provider] of Object.entries(data)) {
|
|
127766
|
-
for (const [modelId, model] of Object.entries(provider.models || {})) {
|
|
127767
|
-
totalModels++;
|
|
127768
|
-
const cost = model.cost || { input: 0, output: 0 };
|
|
127769
|
-
if (cost.input === 0 && cost.output === 0) {
|
|
127770
|
-
continue;
|
|
127771
|
-
}
|
|
127772
|
-
modelsWithPricing++;
|
|
127773
|
-
const normalizedId = this.normalizeModelId(modelId);
|
|
127774
|
-
const pricing = {
|
|
127775
|
-
id: normalizedId,
|
|
127776
|
-
name: model.name || modelId,
|
|
127777
|
-
provider: provider.name || providerId,
|
|
127778
|
-
family: model.family,
|
|
127779
|
-
inputCostPerMillion: cost.input,
|
|
127780
|
-
outputCostPerMillion: cost.output,
|
|
127781
|
-
cacheReadCostPerMillion: cost.cache_read,
|
|
127782
|
-
contextWindow: model.limit?.context,
|
|
127783
|
-
maxOutputTokens: model.limit?.output,
|
|
127784
|
-
lastUpdated: model.last_updated
|
|
127785
|
-
};
|
|
127786
|
-
models.set(modelId, pricing);
|
|
127787
|
-
if (normalizedId !== modelId) {
|
|
127788
|
-
models.set(normalizedId, pricing);
|
|
127789
|
-
}
|
|
127790
|
-
this.addCommonAliases(models, modelId, pricing);
|
|
127791
|
-
}
|
|
127792
|
-
}
|
|
127793
|
-
logger.info(`Loaded ${modelsWithPricing} models with pricing (out of ${totalModels} total)`);
|
|
127794
|
-
return models;
|
|
127795
|
-
} finally {
|
|
127796
|
-
clearTimeout(timeout);
|
|
127797
|
-
}
|
|
127798
|
-
}
|
|
127799
|
-
async getModelPricing(modelId) {
|
|
127800
|
-
const models = await this.fetchAllModels();
|
|
127801
|
-
let pricing = models.get(modelId);
|
|
127802
|
-
if (pricing)
|
|
127803
|
-
return pricing;
|
|
127804
|
-
const normalized = this.normalizeModelId(modelId);
|
|
127805
|
-
pricing = models.get(normalized);
|
|
127806
|
-
if (pricing)
|
|
127807
|
-
return pricing;
|
|
127808
|
-
const lowerModelId = modelId.toLowerCase();
|
|
127809
|
-
for (const [key, value] of models.entries()) {
|
|
127810
|
-
if (key.toLowerCase() === lowerModelId) {
|
|
127811
|
-
return value;
|
|
127812
|
-
}
|
|
127813
|
-
}
|
|
127814
|
-
logger.warn(`Model pricing not found: ${modelId}`);
|
|
127815
|
-
return null;
|
|
127816
|
-
}
|
|
127817
|
-
async searchModels(query) {
|
|
127818
|
-
const models = await this.fetchAllModels();
|
|
127819
|
-
const lowerQuery = query.toLowerCase();
|
|
127820
|
-
const results = [];
|
|
127821
|
-
const seen = new Set;
|
|
127822
|
-
for (const pricing of models.values()) {
|
|
127823
|
-
const key = `${pricing.provider}:${pricing.name}`;
|
|
127824
|
-
if (seen.has(key))
|
|
127825
|
-
continue;
|
|
127826
|
-
seen.add(key);
|
|
127827
|
-
if (pricing.id.toLowerCase().includes(lowerQuery) || pricing.name.toLowerCase().includes(lowerQuery) || pricing.provider.toLowerCase().includes(lowerQuery) || pricing.family && pricing.family.toLowerCase().includes(lowerQuery)) {
|
|
127828
|
-
results.push(pricing);
|
|
127829
|
-
}
|
|
127830
|
-
}
|
|
127831
|
-
results.sort((a, b) => b.inputCostPerMillion - a.inputCostPerMillion);
|
|
127832
|
-
return results;
|
|
127833
|
-
}
|
|
127834
|
-
async getTopExpensiveModels(limit = 10) {
|
|
127835
|
-
const models = await this.fetchAllModels();
|
|
127836
|
-
const unique = new Map;
|
|
127837
|
-
for (const pricing of models.values()) {
|
|
127838
|
-
const key = `${pricing.provider}:${pricing.name}`;
|
|
127839
|
-
if (!unique.has(key)) {
|
|
127840
|
-
unique.set(key, pricing);
|
|
127841
|
-
}
|
|
127842
|
-
}
|
|
127843
|
-
return Array.from(unique.values()).sort((a, b) => b.inputCostPerMillion - a.inputCostPerMillion).slice(0, limit);
|
|
127844
|
-
}
|
|
127845
|
-
async getStatistics() {
|
|
127846
|
-
const models = await this.fetchAllModels();
|
|
127847
|
-
const unique = new Map;
|
|
127848
|
-
const providers = new Set;
|
|
127849
|
-
let totalInputCost = 0;
|
|
127850
|
-
let totalOutputCost = 0;
|
|
127851
|
-
for (const pricing of models.values()) {
|
|
127852
|
-
const key = `${pricing.provider}:${pricing.name}`;
|
|
127853
|
-
if (!unique.has(key)) {
|
|
127854
|
-
unique.set(key, pricing);
|
|
127855
|
-
providers.add(pricing.provider);
|
|
127856
|
-
totalInputCost += pricing.inputCostPerMillion;
|
|
127857
|
-
totalOutputCost += pricing.outputCostPerMillion;
|
|
127468
|
+
return {
|
|
127469
|
+
id: row.id,
|
|
127470
|
+
taskId: row.task_id,
|
|
127471
|
+
taskDescription: row.task_description ?? undefined,
|
|
127472
|
+
agentId: row.agent_id ?? undefined,
|
|
127473
|
+
projectId: row.project_id ?? undefined,
|
|
127474
|
+
state,
|
|
127475
|
+
memoryIds: row.memory_ids ? JSON.parse(row.memory_ids) : [],
|
|
127476
|
+
fileChanges: row.file_changes ? JSON.parse(row.file_changes) : [],
|
|
127477
|
+
checkpointType: row.checkpoint_type,
|
|
127478
|
+
parentCheckpointId: row.parent_checkpoint_id ?? undefined,
|
|
127479
|
+
createdAt: toNum4(row.created_at) ?? Date.now(),
|
|
127480
|
+
expiresAt: toNum4(row.expires_at) ?? undefined
|
|
127481
|
+
};
|
|
127482
|
+
}
|
|
127483
|
+
createCheckpoint(state, options = {}) {
|
|
127484
|
+
const {
|
|
127485
|
+
agentId,
|
|
127486
|
+
projectId,
|
|
127487
|
+
checkpointType = CheckpointType.MANUAL,
|
|
127488
|
+
memoryIds = [],
|
|
127489
|
+
fileChanges = [],
|
|
127490
|
+
parentCheckpointId,
|
|
127491
|
+
ttlMs = 7 * 24 * 60 * 60 * 1000
|
|
127492
|
+
} = options;
|
|
127493
|
+
const id = `ckpt_${checkpointType}_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`;
|
|
127494
|
+
const now2 = Date.now();
|
|
127495
|
+
const expiresAt = now2 + ttlMs;
|
|
127496
|
+
const checkpoint = {
|
|
127497
|
+
id,
|
|
127498
|
+
taskId: state.taskId,
|
|
127499
|
+
taskDescription: state.description,
|
|
127500
|
+
agentId,
|
|
127501
|
+
projectId,
|
|
127502
|
+
state,
|
|
127503
|
+
memoryIds,
|
|
127504
|
+
fileChanges,
|
|
127505
|
+
checkpointType,
|
|
127506
|
+
parentCheckpointId,
|
|
127507
|
+
createdAt: now2,
|
|
127508
|
+
expiresAt
|
|
127509
|
+
};
|
|
127510
|
+
this.mirror.set(id, checkpoint);
|
|
127511
|
+
this.ensureHydrated();
|
|
127512
|
+
this.chainWrite(id, async () => {
|
|
127513
|
+
const prisma2 = this.getClient();
|
|
127514
|
+
const canonicalProjectId = projectId ? await getProjectIdentityAliasResolver().resolve(projectId) : projectId;
|
|
127515
|
+
const compressed = compressState(state);
|
|
127516
|
+
await prisma2.$executeRaw`
|
|
127517
|
+
INSERT INTO task_checkpoints (
|
|
127518
|
+
id, task_id, task_description, agent_id, project_id,
|
|
127519
|
+
state, state_schema_version,
|
|
127520
|
+
memory_ids, file_changes,
|
|
127521
|
+
checkpoint_type, parent_checkpoint_id,
|
|
127522
|
+
created_at, expires_at
|
|
127523
|
+
) VALUES (
|
|
127524
|
+
${id},
|
|
127525
|
+
${state.taskId},
|
|
127526
|
+
${state.description ?? null},
|
|
127527
|
+
${agentId ?? null},
|
|
127528
|
+
${canonicalProjectId ?? null},
|
|
127529
|
+
${compressed},
|
|
127530
|
+
1,
|
|
127531
|
+
${JSON.stringify(memoryIds)},
|
|
127532
|
+
${JSON.stringify(fileChanges)},
|
|
127533
|
+
${checkpointType},
|
|
127534
|
+
${parentCheckpointId ?? null},
|
|
127535
|
+
${now2}::bigint,
|
|
127536
|
+
${expiresAt}::bigint
|
|
127537
|
+
)
|
|
127538
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
127539
|
+
task_id = EXCLUDED.task_id,
|
|
127540
|
+
task_description = EXCLUDED.task_description,
|
|
127541
|
+
agent_id = EXCLUDED.agent_id,
|
|
127542
|
+
project_id = EXCLUDED.project_id,
|
|
127543
|
+
state = EXCLUDED.state,
|
|
127544
|
+
state_schema_version = EXCLUDED.state_schema_version,
|
|
127545
|
+
memory_ids = EXCLUDED.memory_ids,
|
|
127546
|
+
file_changes = EXCLUDED.file_changes,
|
|
127547
|
+
checkpoint_type = EXCLUDED.checkpoint_type,
|
|
127548
|
+
parent_checkpoint_id = EXCLUDED.parent_checkpoint_id,
|
|
127549
|
+
created_at = EXCLUDED.created_at,
|
|
127550
|
+
expires_at = EXCLUDED.expires_at
|
|
127551
|
+
`;
|
|
127552
|
+
});
|
|
127553
|
+
logger.info("Checkpoint created (PG)", {
|
|
127554
|
+
id,
|
|
127555
|
+
taskId: state.taskId,
|
|
127556
|
+
type: checkpointType,
|
|
127557
|
+
compressedBytes: compressState(state).byteLength
|
|
127558
|
+
});
|
|
127559
|
+
return checkpoint;
|
|
127560
|
+
}
|
|
127561
|
+
getCheckpoint(checkpointId) {
|
|
127562
|
+
this.ensureHydrated();
|
|
127563
|
+
return this.mirror.get(checkpointId) ?? null;
|
|
127564
|
+
}
|
|
127565
|
+
listCheckpoints(options = {}) {
|
|
127566
|
+
this.ensureHydrated();
|
|
127567
|
+
return this.applyFilters(this.mirror.values(), options);
|
|
127568
|
+
}
|
|
127569
|
+
listCheckpointsMetadata(options = {}) {
|
|
127570
|
+
this.ensureHydrated();
|
|
127571
|
+
const filtered = this.applyFilters(this.mirror.values(), options);
|
|
127572
|
+
return filtered.map((c) => this.checkpointToMetadata(c));
|
|
127573
|
+
}
|
|
127574
|
+
getCheckpointState(checkpointId) {
|
|
127575
|
+
this.ensureHydrated();
|
|
127576
|
+
const ckpt = this.mirror.get(checkpointId);
|
|
127577
|
+
return ckpt ? ckpt.state : null;
|
|
127578
|
+
}
|
|
127579
|
+
getLatestCheckpoint(taskId) {
|
|
127580
|
+
this.ensureHydrated();
|
|
127581
|
+
const now2 = Date.now();
|
|
127582
|
+
let latest = null;
|
|
127583
|
+
for (const ckpt of this.mirror.values()) {
|
|
127584
|
+
if (ckpt.taskId === taskId && (ckpt.expiresAt == null || ckpt.expiresAt > now2)) {
|
|
127585
|
+
if (!latest || ckpt.createdAt > latest.createdAt) {
|
|
127586
|
+
latest = ckpt;
|
|
127858
127587
|
}
|
|
127859
127588
|
}
|
|
127860
|
-
const modelList = Array.from(unique.values());
|
|
127861
|
-
modelList.sort((a, b) => a.inputCostPerMillion - b.inputCostPerMillion);
|
|
127862
|
-
return {
|
|
127863
|
-
totalModels: modelList.length,
|
|
127864
|
-
totalProviders: providers.size,
|
|
127865
|
-
avgInputCost: modelList.length > 0 ? totalInputCost / modelList.length : 0,
|
|
127866
|
-
avgOutputCost: modelList.length > 0 ? totalOutputCost / modelList.length : 0,
|
|
127867
|
-
cheapestModel: modelList[0] || null,
|
|
127868
|
-
mostExpensiveModel: modelList[modelList.length - 1] || null
|
|
127869
|
-
};
|
|
127870
|
-
}
|
|
127871
|
-
normalizeModelId(modelId) {
|
|
127872
|
-
let normalized = modelId.replace(/^(openai|anthropic|google|meta|mistral|cohere)\//i, "").replace(/^(claude-|gpt-|gemini-|llama-)/i, (match2) => match2.toLowerCase());
|
|
127873
|
-
return normalized;
|
|
127874
127589
|
}
|
|
127875
|
-
|
|
127876
|
-
|
|
127877
|
-
|
|
127878
|
-
|
|
127879
|
-
|
|
127880
|
-
|
|
127881
|
-
|
|
127882
|
-
|
|
127883
|
-
|
|
127884
|
-
|
|
127885
|
-
|
|
127886
|
-
|
|
127887
|
-
|
|
127888
|
-
|
|
127889
|
-
|
|
127890
|
-
|
|
127891
|
-
|
|
127892
|
-
|
|
127893
|
-
|
|
127894
|
-
if (
|
|
127895
|
-
|
|
127896
|
-
|
|
127897
|
-
if (modelId.includes("gemini-1.5-pro")) {
|
|
127898
|
-
models.set("gemini-1.5-pro", pricing);
|
|
127899
|
-
}
|
|
127900
|
-
if (modelId.includes("gemini-1.5-flash")) {
|
|
127901
|
-
models.set("gemini-1.5-flash", pricing);
|
|
127590
|
+
return latest;
|
|
127591
|
+
}
|
|
127592
|
+
deleteCheckpoint(checkpointId) {
|
|
127593
|
+
const existed = this.mirror.has(checkpointId);
|
|
127594
|
+
this.mirror.delete(checkpointId);
|
|
127595
|
+
this.ensureHydrated();
|
|
127596
|
+
this.chainWrite(checkpointId, async () => {
|
|
127597
|
+
const prisma2 = this.getClient();
|
|
127598
|
+
await prisma2.$executeRaw`
|
|
127599
|
+
DELETE FROM task_checkpoints WHERE id = ${checkpointId}
|
|
127600
|
+
`;
|
|
127601
|
+
});
|
|
127602
|
+
return existed;
|
|
127603
|
+
}
|
|
127604
|
+
purgeExpired() {
|
|
127605
|
+
const now2 = Date.now();
|
|
127606
|
+
let count = 0;
|
|
127607
|
+
const toRemove = [];
|
|
127608
|
+
for (const ckpt of this.mirror.values()) {
|
|
127609
|
+
if (ckpt.expiresAt != null && ckpt.expiresAt < now2) {
|
|
127610
|
+
toRemove.push(ckpt.id);
|
|
127611
|
+
count++;
|
|
127902
127612
|
}
|
|
127903
127613
|
}
|
|
127904
|
-
|
|
127905
|
-
this.
|
|
127906
|
-
|
|
127907
|
-
|
|
127614
|
+
for (const id of toRemove)
|
|
127615
|
+
this.mirror.delete(id);
|
|
127616
|
+
if (count > 0) {
|
|
127617
|
+
this.chainWrite("__purge__", async () => {
|
|
127618
|
+
const prisma2 = this.getClient();
|
|
127619
|
+
await prisma2.$executeRaw`
|
|
127620
|
+
DELETE FROM task_checkpoints
|
|
127621
|
+
WHERE expires_at IS NOT NULL AND expires_at < ${now2}::bigint
|
|
127622
|
+
`;
|
|
127623
|
+
});
|
|
127624
|
+
logger.info("Expired checkpoints purged (PG)", { count });
|
|
127908
127625
|
}
|
|
127909
|
-
|
|
127910
|
-
|
|
127911
|
-
|
|
127912
|
-
|
|
127913
|
-
|
|
127914
|
-
|
|
127915
|
-
|
|
127916
|
-
|
|
127917
|
-
|
|
127918
|
-
|
|
127919
|
-
|
|
127920
|
-
|
|
127626
|
+
return count;
|
|
127627
|
+
}
|
|
127628
|
+
async countExistingMemoryIds(memoryIds) {
|
|
127629
|
+
if (memoryIds.length === 0)
|
|
127630
|
+
return [];
|
|
127631
|
+
this.ensureHydrated();
|
|
127632
|
+
const BATCH_SIZE = 1000;
|
|
127633
|
+
const existing = [];
|
|
127634
|
+
try {
|
|
127635
|
+
const prisma2 = this.getClient();
|
|
127636
|
+
for (let i = 0;i < memoryIds.length; i += BATCH_SIZE) {
|
|
127637
|
+
const batch = memoryIds.slice(i, i + BATCH_SIZE);
|
|
127638
|
+
const rows = await prisma2.$queryRaw`
|
|
127639
|
+
SELECT id FROM memories WHERE id IN (${import_prisma4.Prisma.join(batch)})
|
|
127640
|
+
`;
|
|
127641
|
+
for (const row of rows)
|
|
127642
|
+
existing.push(row.id);
|
|
127921
127643
|
}
|
|
127644
|
+
return existing;
|
|
127645
|
+
} catch (e) {
|
|
127646
|
+
logger.warn("countExistingMemoryIds failed (best-effort: assuming all exist)", {
|
|
127647
|
+
error: e.message
|
|
127648
|
+
});
|
|
127649
|
+
return memoryIds;
|
|
127922
127650
|
}
|
|
127923
|
-
|
|
127924
|
-
|
|
127925
|
-
|
|
127651
|
+
}
|
|
127652
|
+
getStats() {
|
|
127653
|
+
this.ensureHydrated();
|
|
127654
|
+
const checkpoints = Array.from(this.mirror.values());
|
|
127655
|
+
const byType = {};
|
|
127656
|
+
let totalSizeBytes = 0;
|
|
127657
|
+
let oldest;
|
|
127658
|
+
for (const c of checkpoints) {
|
|
127659
|
+
byType[c.checkpointType] = (byType[c.checkpointType] ?? 0) + 1;
|
|
127660
|
+
totalSizeBytes += compressState(c.state).byteLength;
|
|
127661
|
+
if (oldest == null || c.createdAt < oldest)
|
|
127662
|
+
oldest = c.createdAt;
|
|
127926
127663
|
}
|
|
127927
|
-
|
|
127928
|
-
|
|
127929
|
-
|
|
127930
|
-
|
|
127931
|
-
|
|
127932
|
-
|
|
127933
|
-
static getInstance() {
|
|
127934
|
-
return this.instance ??= new RedundancyFilter;
|
|
127664
|
+
return {
|
|
127665
|
+
totalCheckpoints: checkpoints.length,
|
|
127666
|
+
byType,
|
|
127667
|
+
totalSizeBytes,
|
|
127668
|
+
oldestCheckpointAge: oldest != null ? Date.now() - oldest : undefined
|
|
127669
|
+
};
|
|
127935
127670
|
}
|
|
127936
|
-
|
|
127937
|
-
|
|
127938
|
-
|
|
127939
|
-
|
|
127940
|
-
|
|
127941
|
-
|
|
127942
|
-
const
|
|
127943
|
-
|
|
127944
|
-
|
|
127671
|
+
ensureReady() {
|
|
127672
|
+
return this.ensureHydrated();
|
|
127673
|
+
}
|
|
127674
|
+
close() {}
|
|
127675
|
+
applyFilters(iter, options) {
|
|
127676
|
+
const { taskId, projectId, checkpointType, includeExpired = false, limit = 20, offset = 0 } = options;
|
|
127677
|
+
const now2 = Date.now();
|
|
127678
|
+
const out = [];
|
|
127679
|
+
for (const c of iter) {
|
|
127680
|
+
if (taskId && c.taskId !== taskId)
|
|
127945
127681
|
continue;
|
|
127946
|
-
|
|
127947
|
-
const vector = new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
|
|
127948
|
-
let norm = 0;
|
|
127949
|
-
for (const value of vector)
|
|
127950
|
-
norm += value * value;
|
|
127951
|
-
norm = Math.sqrt(norm);
|
|
127952
|
-
if (!norm)
|
|
127682
|
+
if (projectId && c.projectId !== projectId)
|
|
127953
127683
|
continue;
|
|
127954
|
-
|
|
127955
|
-
|
|
127956
|
-
|
|
127684
|
+
if (checkpointType && c.checkpointType !== checkpointType)
|
|
127685
|
+
continue;
|
|
127686
|
+
if (!includeExpired && c.expiresAt != null && c.expiresAt <= now2)
|
|
127687
|
+
continue;
|
|
127688
|
+
out.push(c);
|
|
127957
127689
|
}
|
|
127958
|
-
|
|
127959
|
-
|
|
127960
|
-
|
|
127961
|
-
|
|
127962
|
-
|
|
127963
|
-
|
|
127964
|
-
|
|
127965
|
-
|
|
127966
|
-
|
|
127967
|
-
|
|
127968
|
-
|
|
127969
|
-
|
|
127970
|
-
|
|
127971
|
-
|
|
127972
|
-
|
|
127973
|
-
|
|
127974
|
-
|
|
127975
|
-
|
|
127976
|
-
pairs.push({ keepId: keep.row.id, removeId: drop.row.id, similarity, reason: keep.row.importance !== drop.row.importance ? "Higher importance" : "Higher access count" });
|
|
127977
|
-
}
|
|
127978
|
-
return pairs;
|
|
127690
|
+
out.sort((a, b) => b.createdAt - a.createdAt);
|
|
127691
|
+
return out.slice(offset, offset + limit);
|
|
127692
|
+
}
|
|
127693
|
+
checkpointToMetadata(c) {
|
|
127694
|
+
return {
|
|
127695
|
+
id: c.id,
|
|
127696
|
+
taskId: c.taskId,
|
|
127697
|
+
taskDescription: c.taskDescription,
|
|
127698
|
+
agentId: c.agentId,
|
|
127699
|
+
projectId: c.projectId,
|
|
127700
|
+
checkpointType: c.checkpointType,
|
|
127701
|
+
parentCheckpointId: c.parentCheckpointId,
|
|
127702
|
+
createdAt: c.createdAt,
|
|
127703
|
+
expiresAt: c.expiresAt,
|
|
127704
|
+
compressedSizeBytes: compressState(c.state).byteLength,
|
|
127705
|
+
memoryCount: c.memoryIds.length,
|
|
127706
|
+
fileChangeCount: c.fileChanges.length
|
|
127707
|
+
};
|
|
127979
127708
|
}
|
|
127980
|
-
|
|
127981
|
-
|
|
127982
|
-
|
|
127983
|
-
|
|
127984
|
-
|
|
127985
|
-
|
|
127986
|
-
|
|
127987
|
-
|
|
127988
|
-
|
|
127989
|
-
|
|
127990
|
-
|
|
127991
|
-
|
|
127992
|
-
const moved = await tx.$executeRaw`
|
|
127993
|
-
INSERT INTO memory_edges (from_id, to_id, edge_type, weight, metadata, created_at, updated_at)
|
|
127994
|
-
SELECT CASE WHEN from_id = ${pair.removeId} THEN ${pair.keepId} ELSE from_id END,
|
|
127995
|
-
CASE WHEN to_id = ${pair.removeId} THEN ${pair.keepId} ELSE to_id END,
|
|
127996
|
-
edge_type, weight, metadata, NOW(), NOW()
|
|
127997
|
-
FROM memory_edges WHERE (from_id = ${pair.removeId} OR to_id = ${pair.removeId})
|
|
127998
|
-
AND from_id <> ${pair.keepId} AND to_id <> ${pair.keepId}
|
|
127999
|
-
ON CONFLICT (from_id, to_id, edge_type) DO UPDATE SET weight = GREATEST(memory_edges.weight, EXCLUDED.weight), updated_at = NOW()`;
|
|
128000
|
-
await tx.$executeRaw`DELETE FROM memory_edges WHERE from_id = ${pair.removeId} OR to_id = ${pair.removeId}`;
|
|
128001
|
-
await tx.$executeRaw`UPDATE memories SET access_count = access_count + ${rows[0].access_count}, updated_at = NOW() WHERE id = ${pair.keepId}`;
|
|
128002
|
-
await tx.$executeRaw`DELETE FROM memories WHERE id = ${pair.removeId}`;
|
|
128003
|
-
TokenMetrics.getInstance().recordRedundancyFilterSavings(rows[0].content);
|
|
128004
|
-
merged++;
|
|
128005
|
-
edgesTransferred += Number(moved);
|
|
128006
|
-
if (rows[0].access_count > 0)
|
|
128007
|
-
accessCountsBoosted++;
|
|
128008
|
-
}
|
|
127709
|
+
chainWrite(key, fn) {
|
|
127710
|
+
const prev = this.inflight.get(key) ?? Promise.resolve();
|
|
127711
|
+
const next = prev.then(fn).catch((e) => {
|
|
127712
|
+
logger.warn("PgCheckpointStore write failed (best-effort)", {
|
|
127713
|
+
key,
|
|
127714
|
+
error: e.message
|
|
127715
|
+
});
|
|
127716
|
+
});
|
|
127717
|
+
this.inflight.set(key, next);
|
|
127718
|
+
next.then(() => {
|
|
127719
|
+
if (this.inflight.get(key) === next)
|
|
127720
|
+
this.inflight.delete(key);
|
|
128009
127721
|
});
|
|
128010
|
-
logger.info("RedundancyFilter: merge complete", { merged, edgesTransferred, accessCountsBoosted });
|
|
128011
|
-
return { merged, edgesTransferred, accessCountsBoosted };
|
|
128012
127722
|
}
|
|
128013
|
-
async
|
|
128014
|
-
const
|
|
128015
|
-
|
|
128016
|
-
|
|
128017
|
-
|
|
127723
|
+
async __drain() {
|
|
127724
|
+
const pending = Array.from(this.inflight.values());
|
|
127725
|
+
if (pending.length > 0)
|
|
127726
|
+
await Promise.allSettled(pending);
|
|
127727
|
+
await new Promise((r) => setTimeout(r, 10));
|
|
128018
127728
|
}
|
|
128019
|
-
|
|
128020
|
-
|
|
127729
|
+
async __hydrate() {
|
|
127730
|
+
await this.ensureHydrated();
|
|
128021
127731
|
}
|
|
128022
127732
|
}
|
|
128023
|
-
var
|
|
127733
|
+
var import_prisma4, SUPPORTED_CHECKPOINT_STATE_SCHEMA_VERSION = "1.0.0";
|
|
127734
|
+
var init_checkpoint_store_pg = __esm(() => {
|
|
128024
127735
|
init_dist();
|
|
128025
127736
|
init_prisma_client();
|
|
128026
|
-
|
|
127737
|
+
init_alias_resolver();
|
|
127738
|
+
init_schema_version();
|
|
127739
|
+
import_prisma4 = __toESM(require_prisma(), 1);
|
|
128027
127740
|
});
|
|
128028
127741
|
|
|
128029
|
-
// ../../packages/core/dist/services/
|
|
128030
|
-
|
|
128031
|
-
|
|
128032
|
-
|
|
128033
|
-
|
|
128034
|
-
|
|
127742
|
+
// ../../packages/core/dist/services/checkpoint/checkpoint-manager.js
|
|
127743
|
+
var exports_checkpoint_manager = {};
|
|
127744
|
+
__export(exports_checkpoint_manager, {
|
|
127745
|
+
CheckpointManager: () => CheckpointManager
|
|
127746
|
+
});
|
|
127747
|
+
var CheckpointManager;
|
|
127748
|
+
var init_checkpoint_manager = __esm(() => {
|
|
127749
|
+
init_config();
|
|
127750
|
+
init_checkpoint_store_pg();
|
|
127751
|
+
CheckpointManager = class CheckpointManager extends PgCheckpointStore {
|
|
127752
|
+
static instance = null;
|
|
127753
|
+
static getInstance() {
|
|
127754
|
+
requirePostgresDatabaseUrl();
|
|
127755
|
+
return this.instance ??= new CheckpointManager;
|
|
128035
127756
|
}
|
|
128036
|
-
|
|
127757
|
+
async restoreCheckpoint(checkpointId) {
|
|
127758
|
+
const checkpoint = this.getCheckpoint(checkpointId);
|
|
127759
|
+
if (!checkpoint)
|
|
127760
|
+
return null;
|
|
127761
|
+
const existing = new Set(await this.countExistingMemoryIds(checkpoint.memoryIds));
|
|
127762
|
+
const validMemoryIds = checkpoint.memoryIds.filter((id) => existing.has(id));
|
|
127763
|
+
const missingMemoryIds = checkpoint.memoryIds.filter((id) => !existing.has(id));
|
|
127764
|
+
const fileConflicts = [];
|
|
127765
|
+
const restoreInstructions = [
|
|
127766
|
+
`Restore checkpoint ${checkpoint.id} for task ${checkpoint.taskId}.`,
|
|
127767
|
+
missingMemoryIds.length ? `Missing memories: ${missingMemoryIds.join(", ")}.` : "All referenced memories are available."
|
|
127768
|
+
].join(`
|
|
127769
|
+
`);
|
|
127770
|
+
return { checkpoint, validMemoryIds, missingMemoryIds, fileConflicts, restoreInstructions };
|
|
127771
|
+
}
|
|
127772
|
+
};
|
|
127773
|
+
});
|
|
127774
|
+
|
|
127775
|
+
// ../../packages/core/dist/services/scheduler/scheduler-defaults.js
|
|
127776
|
+
function envBool2(key, fallback) {
|
|
127777
|
+
const raw2 = process.env[key];
|
|
127778
|
+
if (raw2 === undefined)
|
|
127779
|
+
return fallback;
|
|
127780
|
+
return raw2 === "true" || raw2 === "1";
|
|
127781
|
+
}
|
|
127782
|
+
function envNum2(key, fallback) {
|
|
127783
|
+
const raw2 = process.env[key];
|
|
127784
|
+
if (raw2 === undefined || raw2 === "")
|
|
127785
|
+
return fallback;
|
|
127786
|
+
const n = Number(raw2);
|
|
127787
|
+
return Number.isFinite(n) && n > 0 ? n : fallback;
|
|
127788
|
+
}
|
|
127789
|
+
function applySafeDefaults(job) {
|
|
127790
|
+
if (!envBool2("MASSA_AI_SCHEDULER_SAFE_DEFAULTS", false)) {
|
|
127791
|
+
return job;
|
|
128037
127792
|
}
|
|
128038
|
-
|
|
128039
|
-
|
|
128040
|
-
const
|
|
128041
|
-
|
|
128042
|
-
|
|
128043
|
-
|
|
128044
|
-
|
|
128045
|
-
|
|
128046
|
-
|
|
128047
|
-
|
|
128048
|
-
|
|
128049
|
-
|
|
128050
|
-
|
|
128051
|
-
|
|
128052
|
-
|
|
128053
|
-
|
|
127793
|
+
if (job.jobKind === "memory-consolidation") {
|
|
127794
|
+
const currentInterval = job.schedule.intervalMs ?? THIRTY_MIN;
|
|
127795
|
+
const safeInterval = Math.max(currentInterval, THIRTY_MIN);
|
|
127796
|
+
return {
|
|
127797
|
+
...job,
|
|
127798
|
+
defaultEnabled: true,
|
|
127799
|
+
schedule: { type: "interval", intervalMs: safeInterval }
|
|
127800
|
+
};
|
|
127801
|
+
}
|
|
127802
|
+
if (job.jobKind === "decay-sweep") {
|
|
127803
|
+
const currentInterval = job.schedule.intervalMs ?? ONE_HOUR;
|
|
127804
|
+
const safeInterval = Math.max(currentInterval, ONE_HOUR);
|
|
127805
|
+
return {
|
|
127806
|
+
...job,
|
|
127807
|
+
defaultEnabled: true,
|
|
127808
|
+
schedule: { type: "interval", intervalMs: safeInterval }
|
|
127809
|
+
};
|
|
127810
|
+
}
|
|
127811
|
+
return job;
|
|
127812
|
+
}
|
|
127813
|
+
function registerDefaultJobs(scheduler) {
|
|
127814
|
+
scheduler.registerHandler("memory-consolidation", async () => {
|
|
127815
|
+
const { memoryConsolidationJob: memoryConsolidationJob2 } = await Promise.resolve().then(() => (init_memory_consolidation_job(), exports_memory_consolidation_job));
|
|
127816
|
+
await memoryConsolidationJob2.consolidate();
|
|
127817
|
+
});
|
|
127818
|
+
scheduler.registerHandler("decay-sweep", async () => {
|
|
127819
|
+
const { memoryConsolidationJob: memoryConsolidationJob2 } = await Promise.resolve().then(() => (init_memory_consolidation_job(), exports_memory_consolidation_job));
|
|
127820
|
+
await memoryConsolidationJob2.consolidate();
|
|
127821
|
+
});
|
|
127822
|
+
scheduler.registerHandler("auto-improve", async (job) => {
|
|
127823
|
+
const { autoImproveJob: autoImproveJob2 } = await Promise.resolve().then(() => (init_auto_improve_job(), exports_auto_improve_job));
|
|
127824
|
+
const projectId = job.payload?.projectId ?? "default";
|
|
127825
|
+
await autoImproveJob2.runOnce(projectId);
|
|
127826
|
+
});
|
|
127827
|
+
scheduler.registerHandler("observation-bridge", async (job) => {
|
|
127828
|
+
const { observationConsolidationJob: observationConsolidationJob2 } = await Promise.resolve().then(() => (init_observation_consolidation_job(), exports_observation_consolidation_job));
|
|
127829
|
+
const projectId = job.payload?.projectId ?? "default";
|
|
127830
|
+
await observationConsolidationJob2.runOnce(projectId);
|
|
127831
|
+
});
|
|
127832
|
+
scheduler.registerHandler("checkpoint-purge", async () => {
|
|
127833
|
+
const { CheckpointManager: CheckpointManager2 } = await Promise.resolve().then(() => (init_checkpoint_manager(), exports_checkpoint_manager));
|
|
127834
|
+
const count = CheckpointManager2.getInstance().purgeExpired();
|
|
127835
|
+
logger.info("Scheduled checkpoint purge completed", { count });
|
|
127836
|
+
});
|
|
127837
|
+
for (const rawDef of DEFAULT_SCHEDULED_JOBS) {
|
|
127838
|
+
const def = applySafeDefaults(rawDef);
|
|
127839
|
+
const enabled = envBool2(def.enableEnvVar, def.defaultEnabled);
|
|
127840
|
+
const intervalMs = envNum2(def.intervalEnvVar, def.schedule.intervalMs ?? THIRTY_MIN);
|
|
127841
|
+
const schedule = { type: "interval", intervalMs };
|
|
127842
|
+
scheduler.registerOrResumeJob({
|
|
127843
|
+
id: def.id,
|
|
127844
|
+
name: def.name,
|
|
127845
|
+
jobKind: def.jobKind,
|
|
127846
|
+
schedule,
|
|
127847
|
+
nextRunAt: 0,
|
|
127848
|
+
enabled
|
|
127849
|
+
});
|
|
127850
|
+
}
|
|
127851
|
+
logger.info("Scheduler default jobs registered", {
|
|
127852
|
+
handlers: scheduler.registeredKinds(),
|
|
127853
|
+
jobs: DEFAULT_SCHEDULED_JOBS.length
|
|
127854
|
+
});
|
|
127855
|
+
}
|
|
127856
|
+
var MIN, THIRTY_MIN, ONE_HOUR, DEFAULT_SCHEDULED_JOBS;
|
|
127857
|
+
var init_scheduler_defaults = __esm(() => {
|
|
127858
|
+
init_dist();
|
|
127859
|
+
MIN = 60 * 1000;
|
|
127860
|
+
THIRTY_MIN = 30 * MIN;
|
|
127861
|
+
ONE_HOUR = 60 * MIN;
|
|
127862
|
+
DEFAULT_SCHEDULED_JOBS = [
|
|
127863
|
+
{
|
|
127864
|
+
id: "scheduled-memory-consolidation",
|
|
127865
|
+
name: "Memory Consolidation (clock)",
|
|
127866
|
+
jobKind: "memory-consolidation",
|
|
127867
|
+
schedule: { type: "interval", intervalMs: THIRTY_MIN },
|
|
127868
|
+
defaultEnabled: false,
|
|
127869
|
+
enableEnvVar: "MASSA_AI_SCHEDULER_CONSOLIDATION_ENABLED",
|
|
127870
|
+
intervalEnvVar: "MASSA_AI_SCHEDULER_CONSOLIDATION_INTERVAL_MS"
|
|
127871
|
+
},
|
|
127872
|
+
{
|
|
127873
|
+
id: "scheduled-decay-sweep",
|
|
127874
|
+
name: "Decay Sweep (clock)",
|
|
127875
|
+
jobKind: "decay-sweep",
|
|
127876
|
+
schedule: { type: "interval", intervalMs: ONE_HOUR },
|
|
127877
|
+
defaultEnabled: false,
|
|
127878
|
+
enableEnvVar: "MASSA_AI_SCHEDULER_DECAY_ENABLED",
|
|
127879
|
+
intervalEnvVar: "MASSA_AI_SCHEDULER_DECAY_INTERVAL_MS"
|
|
127880
|
+
},
|
|
127881
|
+
{
|
|
127882
|
+
id: "scheduled-auto-improve",
|
|
127883
|
+
name: "Auto-Improve (clock)",
|
|
127884
|
+
jobKind: "auto-improve",
|
|
127885
|
+
schedule: { type: "interval", intervalMs: THIRTY_MIN },
|
|
127886
|
+
defaultEnabled: false,
|
|
127887
|
+
enableEnvVar: "MASSA_AI_SCHEDULER_AUTO_IMPROVE_ENABLED",
|
|
127888
|
+
intervalEnvVar: "MASSA_AI_SCHEDULER_AUTO_IMPROVE_INTERVAL_MS"
|
|
127889
|
+
},
|
|
127890
|
+
{
|
|
127891
|
+
id: "scheduled-observation-bridge",
|
|
127892
|
+
name: "Observation Bridge (clock)",
|
|
127893
|
+
jobKind: "observation-bridge",
|
|
127894
|
+
schedule: { type: "interval", intervalMs: THIRTY_MIN },
|
|
127895
|
+
defaultEnabled: false,
|
|
127896
|
+
enableEnvVar: "MASSA_AI_SCHEDULER_OBSERVATION_BRIDGE_ENABLED",
|
|
127897
|
+
intervalEnvVar: "MASSA_AI_SCHEDULER_OBSERVATION_BRIDGE_INTERVAL_MS"
|
|
127898
|
+
},
|
|
127899
|
+
{
|
|
127900
|
+
id: "scheduled-checkpoint-purge",
|
|
127901
|
+
name: "Checkpoint Purge (clock)",
|
|
127902
|
+
jobKind: "checkpoint-purge",
|
|
127903
|
+
schedule: { type: "interval", intervalMs: ONE_HOUR },
|
|
127904
|
+
defaultEnabled: false,
|
|
127905
|
+
enableEnvVar: "MASSA_AI_SCHEDULER_CHECKPOINT_PURGE_ENABLED",
|
|
127906
|
+
intervalEnvVar: "MASSA_AI_SCHEDULER_CHECKPOINT_PURGE_INTERVAL_MS"
|
|
128054
127907
|
}
|
|
128055
|
-
|
|
128056
|
-
|
|
127908
|
+
];
|
|
127909
|
+
});
|
|
127910
|
+
|
|
127911
|
+
// ../../packages/core/dist/services/scheduler/index.js
|
|
127912
|
+
var init_scheduler2 = __esm(() => {
|
|
127913
|
+
init_scheduler();
|
|
127914
|
+
init_scheduler_store_pg();
|
|
127915
|
+
init_scheduler_store_factory();
|
|
127916
|
+
init_scheduler_cron();
|
|
127917
|
+
init_scheduler_defaults();
|
|
127918
|
+
});
|
|
127919
|
+
|
|
127920
|
+
// ../../packages/core/dist/services/pricing/models-dev-client.js
|
|
127921
|
+
import fs11 from "fs/promises";
|
|
127922
|
+
import { existsSync as existsSync4 } from "fs";
|
|
127923
|
+
import path16 from "path";
|
|
127924
|
+
function getModelsDevClient() {
|
|
127925
|
+
if (!clientInstance) {
|
|
127926
|
+
clientInstance = new ModelsDevClient;
|
|
127927
|
+
}
|
|
127928
|
+
return clientInstance;
|
|
127929
|
+
}
|
|
127930
|
+
var ModelsDevClient, clientInstance = null;
|
|
127931
|
+
var init_models_dev_client = __esm(() => {
|
|
127932
|
+
init_dist();
|
|
127933
|
+
init_dist();
|
|
127934
|
+
ModelsDevClient = class ModelsDevClient {
|
|
127935
|
+
static API_URL = "https://models.dev/api.json";
|
|
127936
|
+
static MEMORY_CACHE_TTL = 3600 * 1000;
|
|
127937
|
+
static LOCAL_CACHE_TTL = 24 * 3600 * 1000;
|
|
127938
|
+
static LOCAL_CACHE_FILE = "pricing-cache.json";
|
|
127939
|
+
memoryCache = null;
|
|
127940
|
+
memoryCacheTimestamp = 0;
|
|
127941
|
+
getLocalCachePath() {
|
|
127942
|
+
const dataDir = config.get("dataDir");
|
|
127943
|
+
return path16.join(dataDir, ModelsDevClient.LOCAL_CACHE_FILE);
|
|
128057
127944
|
}
|
|
128058
|
-
|
|
128059
|
-
|
|
128060
|
-
|
|
128061
|
-
|
|
128062
|
-
|
|
128063
|
-
const newAssignments = new Int32Array(items.length);
|
|
128064
|
-
for (let i = 0;i < items.length; i++) {
|
|
128065
|
-
let bestDist = Infinity;
|
|
128066
|
-
let bestCluster = 0;
|
|
128067
|
-
for (let c = 0;c < centroids.length; c++) {
|
|
128068
|
-
const dist = this.euclideanDistanceSq(items[i].vec, centroids[c]);
|
|
128069
|
-
if (dist < bestDist) {
|
|
128070
|
-
bestDist = dist;
|
|
128071
|
-
bestCluster = c;
|
|
128072
|
-
}
|
|
128073
|
-
}
|
|
128074
|
-
newAssignments[i] = bestCluster;
|
|
128075
|
-
}
|
|
128076
|
-
let changed = false;
|
|
128077
|
-
for (let i = 0;i < items.length; i++) {
|
|
128078
|
-
if (newAssignments[i] !== assignments[i]) {
|
|
128079
|
-
changed = true;
|
|
128080
|
-
break;
|
|
127945
|
+
async loadLocalCache() {
|
|
127946
|
+
const cachePath = this.getLocalCachePath();
|
|
127947
|
+
try {
|
|
127948
|
+
if (!existsSync4(cachePath)) {
|
|
127949
|
+
return null;
|
|
128081
127950
|
}
|
|
128082
|
-
|
|
128083
|
-
|
|
128084
|
-
|
|
128085
|
-
|
|
128086
|
-
|
|
128087
|
-
|
|
128088
|
-
|
|
128089
|
-
newCentroids.push(Array.from({ length: dim }, () => 0));
|
|
128090
|
-
counts.push(0);
|
|
128091
|
-
}
|
|
128092
|
-
for (let i = 0;i < items.length; i++) {
|
|
128093
|
-
const c = assignments[i];
|
|
128094
|
-
counts[c]++;
|
|
128095
|
-
for (let d = 0;d < dim; d++) {
|
|
128096
|
-
newCentroids[c][d] += items[i].vec[d];
|
|
127951
|
+
const content = await fs11.readFile(cachePath, "utf-8");
|
|
127952
|
+
const data = JSON.parse(content);
|
|
127953
|
+
const age = Date.now() - data.timestamp;
|
|
127954
|
+
if (age > ModelsDevClient.LOCAL_CACHE_TTL) {
|
|
127955
|
+
logger.debug("Local pricing cache expired", {
|
|
127956
|
+
ageHours: Math.round(age / 3600000)
|
|
127957
|
+
});
|
|
128097
127958
|
}
|
|
128098
|
-
|
|
128099
|
-
|
|
128100
|
-
|
|
128101
|
-
continue;
|
|
128102
|
-
for (let d = 0;d < dim; d++) {
|
|
128103
|
-
newCentroids[c][d] /= counts[c];
|
|
127959
|
+
const models = new Map;
|
|
127960
|
+
for (const [key, value] of Object.entries(data.models)) {
|
|
127961
|
+
models.set(key, value);
|
|
128104
127962
|
}
|
|
127963
|
+
logger.debug("Loaded pricing from local cache", {
|
|
127964
|
+
models: models.size,
|
|
127965
|
+
ageHours: Math.round(age / 3600000)
|
|
127966
|
+
});
|
|
127967
|
+
return models;
|
|
127968
|
+
} catch (error51) {
|
|
127969
|
+
logger.debug("Failed to load local pricing cache", {
|
|
127970
|
+
error: error51.message
|
|
127971
|
+
});
|
|
127972
|
+
return null;
|
|
128105
127973
|
}
|
|
128106
|
-
centroids = newCentroids;
|
|
128107
|
-
}
|
|
128108
|
-
const clusterMap = new Map;
|
|
128109
|
-
for (let i = 0;i < items.length; i++) {
|
|
128110
|
-
const c = assignments[i];
|
|
128111
|
-
if (!clusterMap.has(c))
|
|
128112
|
-
clusterMap.set(c, []);
|
|
128113
|
-
clusterMap.get(c).push(items[i].row);
|
|
128114
|
-
}
|
|
128115
|
-
const clusters = [];
|
|
128116
|
-
let unclustered = 0;
|
|
128117
|
-
for (const [cIdx, members] of clusterMap) {
|
|
128118
|
-
if (members.length < 2) {
|
|
128119
|
-
unclustered += members.length;
|
|
128120
|
-
continue;
|
|
128121
|
-
}
|
|
128122
|
-
const label = this.generateLabel(members);
|
|
128123
|
-
const avgImportance = members.reduce((sum, m) => sum + m.importance, 0) / members.length;
|
|
128124
|
-
const totalAccess = members.reduce((sum, m) => sum + m.access_count, 0);
|
|
128125
|
-
const dominantType = this.getDominantType(members);
|
|
128126
|
-
clusters.push({
|
|
128127
|
-
id: `cluster_${Date.now()}_${cIdx}`,
|
|
128128
|
-
centroid: centroids[cIdx],
|
|
128129
|
-
memberIds: members.map((m) => m.id),
|
|
128130
|
-
label,
|
|
128131
|
-
importance: Math.round(avgImportance * 100) / 100,
|
|
128132
|
-
totalAccess,
|
|
128133
|
-
dominantType
|
|
128134
|
-
});
|
|
128135
127974
|
}
|
|
128136
|
-
|
|
128137
|
-
|
|
128138
|
-
|
|
128139
|
-
|
|
128140
|
-
|
|
128141
|
-
|
|
128142
|
-
|
|
128143
|
-
|
|
128144
|
-
|
|
128145
|
-
|
|
128146
|
-
|
|
128147
|
-
|
|
128148
|
-
|
|
128149
|
-
|
|
127975
|
+
async saveLocalCache(models) {
|
|
127976
|
+
const cachePath = this.getLocalCachePath();
|
|
127977
|
+
try {
|
|
127978
|
+
const dir = path16.dirname(cachePath);
|
|
127979
|
+
await fs11.mkdir(dir, { recursive: true });
|
|
127980
|
+
const data = {
|
|
127981
|
+
timestamp: Date.now(),
|
|
127982
|
+
version: "1.0.0",
|
|
127983
|
+
models: Object.fromEntries(models)
|
|
127984
|
+
};
|
|
127985
|
+
await fs11.writeFile(cachePath, JSON.stringify(data), "utf-8");
|
|
127986
|
+
logger.debug("Saved pricing to local cache", {
|
|
127987
|
+
models: models.size,
|
|
127988
|
+
path: cachePath
|
|
127989
|
+
});
|
|
127990
|
+
} catch (error51) {
|
|
127991
|
+
logger.warn("Failed to save local pricing cache", {
|
|
127992
|
+
error: error51.message
|
|
127993
|
+
});
|
|
128150
127994
|
}
|
|
128151
127995
|
}
|
|
128152
|
-
|
|
128153
|
-
|
|
128154
|
-
|
|
128155
|
-
|
|
128156
|
-
|
|
128157
|
-
|
|
128158
|
-
|
|
128159
|
-
|
|
128160
|
-
|
|
128161
|
-
|
|
128162
|
-
|
|
128163
|
-
|
|
128164
|
-
|
|
128165
|
-
|
|
128166
|
-
|
|
128167
|
-
|
|
128168
|
-
|
|
128169
|
-
|
|
128170
|
-
|
|
128171
|
-
|
|
128172
|
-
|
|
128173
|
-
|
|
128174
|
-
|
|
128175
|
-
|
|
128176
|
-
|
|
128177
|
-
|
|
128178
|
-
|
|
128179
|
-
|
|
128180
|
-
|
|
128181
|
-
|
|
128182
|
-
|
|
128183
|
-
|
|
128184
|
-
|
|
128185
|
-
|
|
128186
|
-
|
|
128187
|
-
|
|
128188
|
-
|
|
128189
|
-
|
|
127996
|
+
getBundledDefaults() {
|
|
127997
|
+
const models = new Map;
|
|
127998
|
+
const defaults3 = [
|
|
127999
|
+
{
|
|
128000
|
+
id: "ollama/nomic-embed-text",
|
|
128001
|
+
name: "Nomic Embed Text",
|
|
128002
|
+
provider: "Ollama",
|
|
128003
|
+
family: "nomic",
|
|
128004
|
+
inputCostPerMillion: 0,
|
|
128005
|
+
outputCostPerMillion: 0,
|
|
128006
|
+
contextWindow: 8192
|
|
128007
|
+
},
|
|
128008
|
+
{
|
|
128009
|
+
id: "ollama/all-minilm",
|
|
128010
|
+
name: "All MiniLM",
|
|
128011
|
+
provider: "Ollama",
|
|
128012
|
+
family: "minilm",
|
|
128013
|
+
inputCostPerMillion: 0,
|
|
128014
|
+
outputCostPerMillion: 0,
|
|
128015
|
+
contextWindow: 512
|
|
128016
|
+
},
|
|
128017
|
+
{
|
|
128018
|
+
id: "ollama/mxbai-embed-large",
|
|
128019
|
+
name: "MxBai Embed Large",
|
|
128020
|
+
provider: "Ollama",
|
|
128021
|
+
family: "mxbai",
|
|
128022
|
+
inputCostPerMillion: 0,
|
|
128023
|
+
outputCostPerMillion: 0,
|
|
128024
|
+
contextWindow: 512
|
|
128025
|
+
},
|
|
128026
|
+
{
|
|
128027
|
+
id: "mistral-embed",
|
|
128028
|
+
name: "Mistral Embed",
|
|
128029
|
+
provider: "Mistral",
|
|
128030
|
+
family: "mistral",
|
|
128031
|
+
inputCostPerMillion: 0.1,
|
|
128032
|
+
outputCostPerMillion: 0,
|
|
128033
|
+
contextWindow: 8192
|
|
128034
|
+
},
|
|
128035
|
+
{
|
|
128036
|
+
id: "codestral-embed",
|
|
128037
|
+
name: "Codestral Embed",
|
|
128038
|
+
provider: "Mistral",
|
|
128039
|
+
family: "codestral",
|
|
128040
|
+
inputCostPerMillion: 0.1,
|
|
128041
|
+
outputCostPerMillion: 0,
|
|
128042
|
+
contextWindow: 32768
|
|
128043
|
+
},
|
|
128044
|
+
{
|
|
128045
|
+
id: "mistral-small-latest",
|
|
128046
|
+
name: "Mistral Small",
|
|
128047
|
+
provider: "Mistral",
|
|
128048
|
+
family: "mistral",
|
|
128049
|
+
inputCostPerMillion: 0.2,
|
|
128050
|
+
outputCostPerMillion: 0.6,
|
|
128051
|
+
contextWindow: 32768,
|
|
128052
|
+
maxOutputTokens: 8192
|
|
128053
|
+
},
|
|
128054
|
+
{
|
|
128055
|
+
id: "text-embedding-3-small",
|
|
128056
|
+
name: "Text Embedding 3 Small",
|
|
128057
|
+
provider: "OpenAI",
|
|
128058
|
+
family: "embedding",
|
|
128059
|
+
inputCostPerMillion: 0.02,
|
|
128060
|
+
outputCostPerMillion: 0,
|
|
128061
|
+
contextWindow: 8191
|
|
128062
|
+
},
|
|
128063
|
+
{
|
|
128064
|
+
id: "gpt-4o-mini",
|
|
128065
|
+
name: "GPT-4o Mini",
|
|
128066
|
+
provider: "OpenAI",
|
|
128067
|
+
family: "gpt-4",
|
|
128068
|
+
inputCostPerMillion: 0.15,
|
|
128069
|
+
outputCostPerMillion: 0.6,
|
|
128070
|
+
contextWindow: 128000,
|
|
128071
|
+
maxOutputTokens: 16384
|
|
128072
|
+
},
|
|
128073
|
+
{
|
|
128074
|
+
id: "claude-3-haiku-20240307",
|
|
128075
|
+
name: "Claude 3 Haiku",
|
|
128076
|
+
provider: "Anthropic",
|
|
128077
|
+
family: "claude-3",
|
|
128078
|
+
inputCostPerMillion: 0.25,
|
|
128079
|
+
outputCostPerMillion: 1.25,
|
|
128080
|
+
contextWindow: 200000,
|
|
128081
|
+
maxOutputTokens: 4096
|
|
128082
|
+
},
|
|
128083
|
+
{
|
|
128084
|
+
id: "claude-3-5-sonnet-20241022",
|
|
128085
|
+
name: "Claude 3.5 Sonnet",
|
|
128086
|
+
provider: "Anthropic",
|
|
128087
|
+
family: "claude-3.5",
|
|
128088
|
+
inputCostPerMillion: 3,
|
|
128089
|
+
outputCostPerMillion: 15,
|
|
128090
|
+
contextWindow: 200000,
|
|
128091
|
+
maxOutputTokens: 8192
|
|
128092
|
+
},
|
|
128093
|
+
{
|
|
128094
|
+
id: "gemini-1.5-flash",
|
|
128095
|
+
name: "Gemini 1.5 Flash",
|
|
128096
|
+
provider: "Google",
|
|
128097
|
+
family: "gemini",
|
|
128098
|
+
inputCostPerMillion: 0.075,
|
|
128099
|
+
outputCostPerMillion: 0.3,
|
|
128100
|
+
contextWindow: 1e6,
|
|
128101
|
+
maxOutputTokens: 8192
|
|
128102
|
+
},
|
|
128103
|
+
{
|
|
128104
|
+
id: "embed-english-v3.0",
|
|
128105
|
+
name: "Embed English v3",
|
|
128106
|
+
provider: "Cohere",
|
|
128107
|
+
family: "embed",
|
|
128108
|
+
inputCostPerMillion: 0.1,
|
|
128109
|
+
outputCostPerMillion: 0,
|
|
128110
|
+
contextWindow: 512
|
|
128190
128111
|
}
|
|
128112
|
+
];
|
|
128113
|
+
for (const pricing of defaults3) {
|
|
128114
|
+
models.set(pricing.id, pricing);
|
|
128191
128115
|
}
|
|
128192
|
-
|
|
128193
|
-
|
|
128194
|
-
}
|
|
128116
|
+
logger.info("Using bundled pricing defaults (offline mode)", {
|
|
128117
|
+
models: models.size
|
|
128118
|
+
});
|
|
128119
|
+
return models;
|
|
128195
128120
|
}
|
|
128196
|
-
|
|
128197
|
-
|
|
128198
|
-
|
|
128199
|
-
|
|
128200
|
-
|
|
128201
|
-
|
|
128202
|
-
|
|
128203
|
-
|
|
128204
|
-
|
|
128205
|
-
|
|
128206
|
-
|
|
128207
|
-
|
|
128208
|
-
|
|
128209
|
-
|
|
128210
|
-
|
|
128211
|
-
|
|
128212
|
-
|
|
128213
|
-
|
|
128214
|
-
|
|
128215
|
-
|
|
128216
|
-
|
|
128217
|
-
|
|
128218
|
-
|
|
128219
|
-
|
|
128220
|
-
|
|
128221
|
-
|
|
128222
|
-
|
|
128223
|
-
|
|
128224
|
-
|
|
128225
|
-
|
|
128226
|
-
|
|
128227
|
-
|
|
128228
|
-
|
|
128229
|
-
|
|
128230
|
-
|
|
128231
|
-
|
|
128232
|
-
|
|
128233
|
-
|
|
128234
|
-
|
|
128235
|
-
|
|
128236
|
-
|
|
128237
|
-
|
|
128238
|
-
|
|
128239
|
-
|
|
128240
|
-
|
|
128241
|
-
|
|
128242
|
-
|
|
128243
|
-
|
|
128244
|
-
|
|
128245
|
-
|
|
128246
|
-
|
|
128247
|
-
|
|
128248
|
-
|
|
128249
|
-
|
|
128250
|
-
|
|
128251
|
-
|
|
128252
|
-
|
|
128253
|
-
|
|
128254
|
-
|
|
128255
|
-
|
|
128256
|
-
|
|
128257
|
-
|
|
128258
|
-
|
|
128259
|
-
|
|
128260
|
-
|
|
128261
|
-
|
|
128262
|
-
|
|
128263
|
-
|
|
128264
|
-
|
|
128265
|
-
|
|
128266
|
-
|
|
128267
|
-
|
|
128268
|
-
|
|
128269
|
-
|
|
128270
|
-
|
|
128271
|
-
|
|
128272
|
-
|
|
128273
|
-
|
|
128274
|
-
|
|
128275
|
-
|
|
128276
|
-
|
|
128277
|
-
|
|
128278
|
-
|
|
128279
|
-
|
|
128280
|
-
|
|
128281
|
-
|
|
128282
|
-
|
|
128283
|
-
|
|
128284
|
-
|
|
128285
|
-
|
|
128286
|
-
|
|
128287
|
-
|
|
128288
|
-
|
|
128289
|
-
|
|
128290
|
-
|
|
128291
|
-
|
|
128292
|
-
|
|
128293
|
-
|
|
128294
|
-
|
|
128295
|
-
|
|
128296
|
-
"these",
|
|
128297
|
-
"those",
|
|
128298
|
-
"i",
|
|
128299
|
-
"me",
|
|
128300
|
-
"my",
|
|
128301
|
-
"myself",
|
|
128302
|
-
"we",
|
|
128303
|
-
"our",
|
|
128304
|
-
"ours",
|
|
128305
|
-
"you",
|
|
128306
|
-
"your",
|
|
128307
|
-
"he",
|
|
128308
|
-
"him",
|
|
128309
|
-
"his",
|
|
128310
|
-
"she",
|
|
128311
|
-
"her",
|
|
128312
|
-
"they",
|
|
128313
|
-
"them",
|
|
128314
|
-
"their",
|
|
128315
|
-
"de",
|
|
128316
|
-
"da",
|
|
128317
|
-
"do",
|
|
128318
|
-
"das",
|
|
128319
|
-
"dos",
|
|
128320
|
-
"em",
|
|
128321
|
-
"no",
|
|
128322
|
-
"na",
|
|
128323
|
-
"nos",
|
|
128324
|
-
"nas",
|
|
128325
|
-
"um",
|
|
128326
|
-
"uma",
|
|
128327
|
-
"uns",
|
|
128328
|
-
"umas",
|
|
128329
|
-
"para",
|
|
128330
|
-
"com",
|
|
128331
|
-
"por",
|
|
128332
|
-
"que",
|
|
128333
|
-
"se",
|
|
128334
|
-
"como",
|
|
128335
|
-
"mas",
|
|
128336
|
-
"ou",
|
|
128337
|
-
"quando",
|
|
128338
|
-
"mais",
|
|
128339
|
-
"tamb\xE9m",
|
|
128340
|
-
"j\xE1",
|
|
128341
|
-
"ainda",
|
|
128342
|
-
"sobre",
|
|
128343
|
-
"entre",
|
|
128344
|
-
"at\xE9",
|
|
128345
|
-
"sem",
|
|
128346
|
-
"sob",
|
|
128347
|
-
"esse",
|
|
128348
|
-
"essa",
|
|
128349
|
-
"este",
|
|
128350
|
-
"esta",
|
|
128351
|
-
"aquele",
|
|
128352
|
-
"aquela",
|
|
128353
|
-
"ele",
|
|
128354
|
-
"ela",
|
|
128355
|
-
"eles",
|
|
128356
|
-
"elas",
|
|
128357
|
-
"n\xF3s",
|
|
128358
|
-
"eu",
|
|
128359
|
-
"tu",
|
|
128360
|
-
"voc\xEA",
|
|
128361
|
-
"voc\xEAs",
|
|
128362
|
-
"meu",
|
|
128363
|
-
"minha",
|
|
128364
|
-
"seu",
|
|
128365
|
-
"sua"
|
|
128366
|
-
]);
|
|
128367
|
-
const wordFreq = new Map;
|
|
128368
|
-
for (const member of members) {
|
|
128369
|
-
const words = member.content.toLowerCase().replace(/[^a-z\u00E1\u00E0\u00E2\u00E3\u00E9\u00E8\u00EA\u00ED\u00EF\u00F3\u00F4\u00F5\u00FA\u00FC\u00E7\s-]/g, " ").split(/\s+/).filter((w) => w.length > 2 && !stopWords.has(w));
|
|
128121
|
+
async fetchAllModels() {
|
|
128122
|
+
if (this.memoryCache && Date.now() - this.memoryCacheTimestamp < ModelsDevClient.MEMORY_CACHE_TTL) {
|
|
128123
|
+
logger.debug("Using in-memory pricing cache");
|
|
128124
|
+
return this.memoryCache;
|
|
128125
|
+
}
|
|
128126
|
+
const localCache = await this.loadLocalCache();
|
|
128127
|
+
let remoteModels = null;
|
|
128128
|
+
try {
|
|
128129
|
+
remoteModels = await this.fetchRemote();
|
|
128130
|
+
await this.saveLocalCache(remoteModels);
|
|
128131
|
+
} catch (error51) {
|
|
128132
|
+
logger.debug("Remote pricing fetch failed (offline?)", {
|
|
128133
|
+
error: error51.message
|
|
128134
|
+
});
|
|
128135
|
+
}
|
|
128136
|
+
let result;
|
|
128137
|
+
if (remoteModels) {
|
|
128138
|
+
result = remoteModels;
|
|
128139
|
+
logger.debug("Using remote pricing data");
|
|
128140
|
+
} else if (localCache) {
|
|
128141
|
+
result = localCache;
|
|
128142
|
+
logger.info("Using local pricing cache (offline mode)");
|
|
128143
|
+
} else {
|
|
128144
|
+
result = this.getBundledDefaults();
|
|
128145
|
+
}
|
|
128146
|
+
this.memoryCache = result;
|
|
128147
|
+
this.memoryCacheTimestamp = Date.now();
|
|
128148
|
+
return result;
|
|
128149
|
+
}
|
|
128150
|
+
async fetchRemote() {
|
|
128151
|
+
const controller = new AbortController;
|
|
128152
|
+
const timeout = setTimeout(() => controller.abort(), 1e4);
|
|
128153
|
+
try {
|
|
128154
|
+
logger.info("Fetching pricing data from models.dev API");
|
|
128155
|
+
const response = await fetch(ModelsDevClient.API_URL, {
|
|
128156
|
+
signal: controller.signal
|
|
128157
|
+
});
|
|
128158
|
+
if (!response.ok) {
|
|
128159
|
+
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
128160
|
+
}
|
|
128161
|
+
const data = await response.json();
|
|
128162
|
+
const models = new Map;
|
|
128163
|
+
let totalModels = 0;
|
|
128164
|
+
let modelsWithPricing = 0;
|
|
128165
|
+
for (const [providerId, provider] of Object.entries(data)) {
|
|
128166
|
+
for (const [modelId, model] of Object.entries(provider.models || {})) {
|
|
128167
|
+
totalModels++;
|
|
128168
|
+
const cost = model.cost || { input: 0, output: 0 };
|
|
128169
|
+
if (cost.input === 0 && cost.output === 0) {
|
|
128170
|
+
continue;
|
|
128171
|
+
}
|
|
128172
|
+
modelsWithPricing++;
|
|
128173
|
+
const normalizedId = this.normalizeModelId(modelId);
|
|
128174
|
+
const pricing = {
|
|
128175
|
+
id: normalizedId,
|
|
128176
|
+
name: model.name || modelId,
|
|
128177
|
+
provider: provider.name || providerId,
|
|
128178
|
+
family: model.family,
|
|
128179
|
+
inputCostPerMillion: cost.input,
|
|
128180
|
+
outputCostPerMillion: cost.output,
|
|
128181
|
+
cacheReadCostPerMillion: cost.cache_read,
|
|
128182
|
+
contextWindow: model.limit?.context,
|
|
128183
|
+
maxOutputTokens: model.limit?.output,
|
|
128184
|
+
lastUpdated: model.last_updated
|
|
128185
|
+
};
|
|
128186
|
+
models.set(modelId, pricing);
|
|
128187
|
+
if (normalizedId !== modelId) {
|
|
128188
|
+
models.set(normalizedId, pricing);
|
|
128189
|
+
}
|
|
128190
|
+
this.addCommonAliases(models, modelId, pricing);
|
|
128191
|
+
}
|
|
128192
|
+
}
|
|
128193
|
+
logger.info(`Loaded ${modelsWithPricing} models with pricing (out of ${totalModels} total)`);
|
|
128194
|
+
return models;
|
|
128195
|
+
} finally {
|
|
128196
|
+
clearTimeout(timeout);
|
|
128197
|
+
}
|
|
128198
|
+
}
|
|
128199
|
+
async getModelPricing(modelId) {
|
|
128200
|
+
const models = await this.fetchAllModels();
|
|
128201
|
+
let pricing = models.get(modelId);
|
|
128202
|
+
if (pricing)
|
|
128203
|
+
return pricing;
|
|
128204
|
+
const normalized = this.normalizeModelId(modelId);
|
|
128205
|
+
pricing = models.get(normalized);
|
|
128206
|
+
if (pricing)
|
|
128207
|
+
return pricing;
|
|
128208
|
+
const lowerModelId = modelId.toLowerCase();
|
|
128209
|
+
for (const [key, value] of models.entries()) {
|
|
128210
|
+
if (key.toLowerCase() === lowerModelId) {
|
|
128211
|
+
return value;
|
|
128212
|
+
}
|
|
128213
|
+
}
|
|
128214
|
+
logger.warn(`Model pricing not found: ${modelId}`);
|
|
128215
|
+
return null;
|
|
128216
|
+
}
|
|
128217
|
+
async searchModels(query) {
|
|
128218
|
+
const models = await this.fetchAllModels();
|
|
128219
|
+
const lowerQuery = query.toLowerCase();
|
|
128220
|
+
const results = [];
|
|
128370
128221
|
const seen = new Set;
|
|
128371
|
-
for (const
|
|
128372
|
-
|
|
128222
|
+
for (const pricing of models.values()) {
|
|
128223
|
+
const key = `${pricing.provider}:${pricing.name}`;
|
|
128224
|
+
if (seen.has(key))
|
|
128373
128225
|
continue;
|
|
128374
|
-
seen.add(
|
|
128375
|
-
|
|
128226
|
+
seen.add(key);
|
|
128227
|
+
if (pricing.id.toLowerCase().includes(lowerQuery) || pricing.name.toLowerCase().includes(lowerQuery) || pricing.provider.toLowerCase().includes(lowerQuery) || pricing.family && pricing.family.toLowerCase().includes(lowerQuery)) {
|
|
128228
|
+
results.push(pricing);
|
|
128229
|
+
}
|
|
128376
128230
|
}
|
|
128231
|
+
results.sort((a, b) => b.inputCostPerMillion - a.inputCostPerMillion);
|
|
128232
|
+
return results;
|
|
128377
128233
|
}
|
|
128378
|
-
|
|
128379
|
-
|
|
128234
|
+
async getTopExpensiveModels(limit = 10) {
|
|
128235
|
+
const models = await this.fetchAllModels();
|
|
128236
|
+
const unique = new Map;
|
|
128237
|
+
for (const pricing of models.values()) {
|
|
128238
|
+
const key = `${pricing.provider}:${pricing.name}`;
|
|
128239
|
+
if (!unique.has(key)) {
|
|
128240
|
+
unique.set(key, pricing);
|
|
128241
|
+
}
|
|
128242
|
+
}
|
|
128243
|
+
return Array.from(unique.values()).sort((a, b) => b.inputCostPerMillion - a.inputCostPerMillion).slice(0, limit);
|
|
128244
|
+
}
|
|
128245
|
+
async getStatistics() {
|
|
128246
|
+
const models = await this.fetchAllModels();
|
|
128247
|
+
const unique = new Map;
|
|
128248
|
+
const providers = new Set;
|
|
128249
|
+
let totalInputCost = 0;
|
|
128250
|
+
let totalOutputCost = 0;
|
|
128251
|
+
for (const pricing of models.values()) {
|
|
128252
|
+
const key = `${pricing.provider}:${pricing.name}`;
|
|
128253
|
+
if (!unique.has(key)) {
|
|
128254
|
+
unique.set(key, pricing);
|
|
128255
|
+
providers.add(pricing.provider);
|
|
128256
|
+
totalInputCost += pricing.inputCostPerMillion;
|
|
128257
|
+
totalOutputCost += pricing.outputCostPerMillion;
|
|
128258
|
+
}
|
|
128259
|
+
}
|
|
128260
|
+
const modelList = Array.from(unique.values());
|
|
128261
|
+
modelList.sort((a, b) => a.inputCostPerMillion - b.inputCostPerMillion);
|
|
128262
|
+
return {
|
|
128263
|
+
totalModels: modelList.length,
|
|
128264
|
+
totalProviders: providers.size,
|
|
128265
|
+
avgInputCost: modelList.length > 0 ? totalInputCost / modelList.length : 0,
|
|
128266
|
+
avgOutputCost: modelList.length > 0 ? totalOutputCost / modelList.length : 0,
|
|
128267
|
+
cheapestModel: modelList[0] || null,
|
|
128268
|
+
mostExpensiveModel: modelList[modelList.length - 1] || null
|
|
128269
|
+
};
|
|
128270
|
+
}
|
|
128271
|
+
normalizeModelId(modelId) {
|
|
128272
|
+
let normalized = modelId.replace(/^(openai|anthropic|google|meta|mistral|cohere)\//i, "").replace(/^(claude-|gpt-|gemini-|llama-)/i, (match2) => match2.toLowerCase());
|
|
128273
|
+
return normalized;
|
|
128274
|
+
}
|
|
128275
|
+
addCommonAliases(models, modelId, pricing) {
|
|
128276
|
+
if (modelId.includes("gpt-4") && !modelId.includes("turbo")) {
|
|
128277
|
+
models.set("gpt-4", pricing);
|
|
128278
|
+
}
|
|
128279
|
+
if (modelId.includes("gpt-4-turbo")) {
|
|
128280
|
+
models.set("gpt-4-turbo", pricing);
|
|
128281
|
+
}
|
|
128282
|
+
if (modelId.includes("gpt-3.5-turbo")) {
|
|
128283
|
+
models.set("gpt-3.5-turbo", pricing);
|
|
128284
|
+
}
|
|
128285
|
+
if (modelId.includes("claude-3-opus")) {
|
|
128286
|
+
models.set("claude-3-opus", pricing);
|
|
128287
|
+
}
|
|
128288
|
+
if (modelId.includes("claude-3-sonnet")) {
|
|
128289
|
+
models.set("claude-3-sonnet", pricing);
|
|
128290
|
+
}
|
|
128291
|
+
if (modelId.includes("claude-3-haiku")) {
|
|
128292
|
+
models.set("claude-3-haiku", pricing);
|
|
128293
|
+
}
|
|
128294
|
+
if (modelId.includes("gemini-pro")) {
|
|
128295
|
+
models.set("gemini-pro", pricing);
|
|
128296
|
+
}
|
|
128297
|
+
if (modelId.includes("gemini-1.5-pro")) {
|
|
128298
|
+
models.set("gemini-1.5-pro", pricing);
|
|
128299
|
+
}
|
|
128300
|
+
if (modelId.includes("gemini-1.5-flash")) {
|
|
128301
|
+
models.set("gemini-1.5-flash", pricing);
|
|
128302
|
+
}
|
|
128303
|
+
}
|
|
128304
|
+
clearCache() {
|
|
128305
|
+
this.memoryCache = null;
|
|
128306
|
+
this.memoryCacheTimestamp = 0;
|
|
128307
|
+
logger.debug("Models.dev pricing memory cache cleared");
|
|
128308
|
+
}
|
|
128309
|
+
async clearAllCaches() {
|
|
128310
|
+
this.clearCache();
|
|
128311
|
+
const cachePath = this.getLocalCachePath();
|
|
128312
|
+
try {
|
|
128313
|
+
if (existsSync4(cachePath)) {
|
|
128314
|
+
await fs11.unlink(cachePath);
|
|
128315
|
+
logger.debug("Local pricing cache file deleted");
|
|
128316
|
+
}
|
|
128317
|
+
} catch (error51) {
|
|
128318
|
+
logger.warn("Failed to delete local pricing cache", {
|
|
128319
|
+
error: error51.message
|
|
128320
|
+
});
|
|
128321
|
+
}
|
|
128322
|
+
}
|
|
128323
|
+
async hasLocalCache() {
|
|
128324
|
+
const cachePath = this.getLocalCachePath();
|
|
128325
|
+
return existsSync4(cachePath);
|
|
128326
|
+
}
|
|
128327
|
+
};
|
|
128328
|
+
});
|
|
128329
|
+
|
|
128330
|
+
// ../../packages/core/dist/services/memory/redundancy-filter.js
|
|
128331
|
+
class RedundancyFilter {
|
|
128332
|
+
static instance = null;
|
|
128333
|
+
static getInstance() {
|
|
128334
|
+
return this.instance ??= new RedundancyFilter;
|
|
128380
128335
|
}
|
|
128381
|
-
|
|
128382
|
-
const
|
|
128383
|
-
|
|
128384
|
-
|
|
128336
|
+
async findDuplicates(threshold = 0.95, scanLimit = 300) {
|
|
128337
|
+
const rows = await getPrismaClient2().$queryRaw`
|
|
128338
|
+
SELECT id, content, type, level, importance, array_to_json(tags)::text AS tags, embedding,
|
|
128339
|
+
created_at, updated_at, access_count, user_id, session_id, project_id, agent_id
|
|
128340
|
+
FROM memories WHERE embedding IS NOT NULL AND deleted_at IS NULL
|
|
128341
|
+
ORDER BY created_at DESC LIMIT ${scanLimit}`;
|
|
128342
|
+
const groups = new Map;
|
|
128343
|
+
for (const row of rows) {
|
|
128344
|
+
if (!row.embedding)
|
|
128345
|
+
continue;
|
|
128346
|
+
const bytes = Buffer.from(row.embedding);
|
|
128347
|
+
const vector = new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
|
|
128348
|
+
let norm = 0;
|
|
128349
|
+
for (const value of vector)
|
|
128350
|
+
norm += value * value;
|
|
128351
|
+
norm = Math.sqrt(norm);
|
|
128352
|
+
if (!norm)
|
|
128353
|
+
continue;
|
|
128354
|
+
const group = groups.get(row.type) ?? [];
|
|
128355
|
+
group.push({ row, vector, norm });
|
|
128356
|
+
groups.set(row.type, group);
|
|
128385
128357
|
}
|
|
128386
|
-
|
|
128387
|
-
|
|
128388
|
-
for (const
|
|
128389
|
-
|
|
128390
|
-
|
|
128391
|
-
|
|
128358
|
+
const pairs = [];
|
|
128359
|
+
const removed = new Set;
|
|
128360
|
+
for (const group of groups.values())
|
|
128361
|
+
for (let left = 0;left < group.length; left++)
|
|
128362
|
+
for (let right = left + 1;right < group.length; right++) {
|
|
128363
|
+
const a = group[left];
|
|
128364
|
+
const b = group[right];
|
|
128365
|
+
if (removed.has(a.row.id) || removed.has(b.row.id) || a.vector.length !== b.vector.length)
|
|
128366
|
+
continue;
|
|
128367
|
+
let dot = 0;
|
|
128368
|
+
for (let index = 0;index < a.vector.length; index++)
|
|
128369
|
+
dot += a.vector[index] * b.vector[index];
|
|
128370
|
+
const similarity = dot / (a.norm * b.norm);
|
|
128371
|
+
if (similarity < threshold)
|
|
128372
|
+
continue;
|
|
128373
|
+
const keep = a.row.importance !== b.row.importance ? a.row.importance > b.row.importance ? a : b : a.row.access_count >= b.row.access_count ? a : b;
|
|
128374
|
+
const drop = keep === a ? b : a;
|
|
128375
|
+
removed.add(drop.row.id);
|
|
128376
|
+
pairs.push({ keepId: keep.row.id, removeId: drop.row.id, similarity, reason: keep.row.importance !== drop.row.importance ? "Higher importance" : "Higher access count" });
|
|
128377
|
+
}
|
|
128378
|
+
return pairs;
|
|
128379
|
+
}
|
|
128380
|
+
async mergeDuplicates(pairs) {
|
|
128381
|
+
if (!pairs.length)
|
|
128382
|
+
return { merged: 0, edgesTransferred: 0, accessCountsBoosted: 0 };
|
|
128383
|
+
const prisma2 = getPrismaClient2();
|
|
128384
|
+
let merged = 0;
|
|
128385
|
+
let edgesTransferred = 0;
|
|
128386
|
+
let accessCountsBoosted = 0;
|
|
128387
|
+
await prisma2.$transaction(async (tx) => {
|
|
128388
|
+
for (const pair of pairs) {
|
|
128389
|
+
const rows = await tx.$queryRaw`SELECT content, access_count FROM memories WHERE id = ${pair.removeId} FOR UPDATE`;
|
|
128390
|
+
if (!rows[0])
|
|
128391
|
+
continue;
|
|
128392
|
+
const moved = await tx.$executeRaw`
|
|
128393
|
+
INSERT INTO memory_edges (from_id, to_id, edge_type, weight, metadata, created_at, updated_at)
|
|
128394
|
+
SELECT CASE WHEN from_id = ${pair.removeId} THEN ${pair.keepId} ELSE from_id END,
|
|
128395
|
+
CASE WHEN to_id = ${pair.removeId} THEN ${pair.keepId} ELSE to_id END,
|
|
128396
|
+
edge_type, weight, metadata, NOW(), NOW()
|
|
128397
|
+
FROM memory_edges WHERE (from_id = ${pair.removeId} OR to_id = ${pair.removeId})
|
|
128398
|
+
AND from_id <> ${pair.keepId} AND to_id <> ${pair.keepId}
|
|
128399
|
+
ON CONFLICT (from_id, to_id, edge_type) DO UPDATE SET weight = GREATEST(memory_edges.weight, EXCLUDED.weight), updated_at = NOW()`;
|
|
128400
|
+
await tx.$executeRaw`DELETE FROM memory_edges WHERE from_id = ${pair.removeId} OR to_id = ${pair.removeId}`;
|
|
128401
|
+
await tx.$executeRaw`UPDATE memories SET access_count = access_count + ${rows[0].access_count}, updated_at = NOW() WHERE id = ${pair.keepId}`;
|
|
128402
|
+
await tx.$executeRaw`DELETE FROM memories WHERE id = ${pair.removeId}`;
|
|
128403
|
+
TokenMetrics.getInstance().recordRedundancyFilterSavings(rows[0].content);
|
|
128404
|
+
merged++;
|
|
128405
|
+
edgesTransferred += Number(moved);
|
|
128406
|
+
if (rows[0].access_count > 0)
|
|
128407
|
+
accessCountsBoosted++;
|
|
128392
128408
|
}
|
|
128393
|
-
}
|
|
128394
|
-
|
|
128409
|
+
});
|
|
128410
|
+
logger.info("RedundancyFilter: merge complete", { merged, edgesTransferred, accessCountsBoosted });
|
|
128411
|
+
return { merged, edgesTransferred, accessCountsBoosted };
|
|
128395
128412
|
}
|
|
128396
|
-
|
|
128397
|
-
|
|
128398
|
-
|
|
128399
|
-
|
|
128400
|
-
|
|
128401
|
-
}
|
|
128402
|
-
return sum;
|
|
128413
|
+
async runCleanup(threshold = 0.95) {
|
|
128414
|
+
const start = Date.now();
|
|
128415
|
+
const pairs = await this.findDuplicates(threshold);
|
|
128416
|
+
const result = await this.mergeDuplicates(pairs);
|
|
128417
|
+
return { duplicatesFound: pairs.length, merged: result.merged, edgesTransferred: result.edgesTransferred, durationMs: Date.now() - start };
|
|
128403
128418
|
}
|
|
128404
128419
|
close() {
|
|
128405
|
-
|
|
128420
|
+
RedundancyFilter.instance = null;
|
|
128406
128421
|
}
|
|
128407
128422
|
}
|
|
128408
|
-
var
|
|
128423
|
+
var init_redundancy_filter = __esm(() => {
|
|
128409
128424
|
init_dist();
|
|
128410
128425
|
init_prisma_client();
|
|
128426
|
+
init_token_metrics();
|
|
128411
128427
|
});
|
|
128412
128428
|
|
|
128413
|
-
// ../../packages/core/dist/services/
|
|
128414
|
-
|
|
128415
|
-
|
|
128416
|
-
|
|
128417
|
-
|
|
128418
|
-
|
|
128419
|
-
function compressState(state) {
|
|
128420
|
-
const json3 = JSON.stringify(state);
|
|
128421
|
-
return Buffer.from(Bun.deflateSync(Buffer.from(json3, "utf-8")));
|
|
128422
|
-
}
|
|
128423
|
-
function decompressState(data) {
|
|
128424
|
-
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
128425
|
-
const inflated = Bun.inflateSync(new Uint8Array(buf));
|
|
128426
|
-
return JSON.parse(Buffer.from(inflated).toString("utf-8"));
|
|
128427
|
-
}
|
|
128428
|
-
|
|
128429
|
-
class PgCheckpointStore {
|
|
128430
|
-
prisma;
|
|
128431
|
-
mirror = new Map;
|
|
128432
|
-
hydrated = false;
|
|
128433
|
-
hydrating = null;
|
|
128434
|
-
hydrateFailedAt = 0;
|
|
128435
|
-
static HYDRATE_RETRY_MS = 30000;
|
|
128436
|
-
inflight = new Map;
|
|
128437
|
-
getClient() {
|
|
128438
|
-
if (!this.prisma)
|
|
128439
|
-
this.prisma = getPrismaClient2();
|
|
128440
|
-
return this.prisma;
|
|
128441
|
-
}
|
|
128442
|
-
ensureHydrated() {
|
|
128443
|
-
if (this.hydrated)
|
|
128444
|
-
return Promise.resolve();
|
|
128445
|
-
if (this.hydrating)
|
|
128446
|
-
return this.hydrating;
|
|
128447
|
-
if (this.hydrateFailedAt > 0 && Date.now() - this.hydrateFailedAt < PgCheckpointStore.HYDRATE_RETRY_MS) {
|
|
128448
|
-
return Promise.resolve();
|
|
128429
|
+
// ../../packages/core/dist/services/memory/memory-clustering.js
|
|
128430
|
+
class MemoryClustering {
|
|
128431
|
+
static instance = null;
|
|
128432
|
+
static getInstance() {
|
|
128433
|
+
if (!MemoryClustering.instance) {
|
|
128434
|
+
MemoryClustering.instance = new MemoryClustering;
|
|
128449
128435
|
}
|
|
128450
|
-
|
|
128451
|
-
try {
|
|
128452
|
-
const prisma2 = this.getClient();
|
|
128453
|
-
const rows = await prisma2.$queryRaw`
|
|
128454
|
-
SELECT * FROM task_checkpoints
|
|
128455
|
-
`;
|
|
128456
|
-
const next = new Map;
|
|
128457
|
-
const dbIds = new Set;
|
|
128458
|
-
for (const row of rows) {
|
|
128459
|
-
dbIds.add(row.id);
|
|
128460
|
-
next.set(row.id, this.rowToCheckpoint(row));
|
|
128461
|
-
}
|
|
128462
|
-
for (const [id, existing] of this.mirror) {
|
|
128463
|
-
if (!dbIds.has(id))
|
|
128464
|
-
next.set(id, existing);
|
|
128465
|
-
}
|
|
128466
|
-
this.mirror = next;
|
|
128467
|
-
this.hydrated = true;
|
|
128468
|
-
this.hydrateFailedAt = 0;
|
|
128469
|
-
logger.info("PgCheckpointStore hydrated", {
|
|
128470
|
-
rows: this.mirror.size
|
|
128471
|
-
});
|
|
128472
|
-
} catch (e) {
|
|
128473
|
-
this.hydrateFailedAt = Date.now();
|
|
128474
|
-
logger.warn("PgCheckpointStore hydrate failed (best-effort)", {
|
|
128475
|
-
error: e.message
|
|
128476
|
-
});
|
|
128477
|
-
} finally {
|
|
128478
|
-
this.hydrating = null;
|
|
128479
|
-
}
|
|
128480
|
-
})();
|
|
128481
|
-
return this.hydrating;
|
|
128436
|
+
return MemoryClustering.instance;
|
|
128482
128437
|
}
|
|
128483
|
-
|
|
128484
|
-
|
|
128485
|
-
const
|
|
128486
|
-
|
|
128487
|
-
|
|
128488
|
-
|
|
128489
|
-
|
|
128438
|
+
constructor() {}
|
|
128439
|
+
async clusterMemories(k, maxIter = 20, maxMemories = 500) {
|
|
128440
|
+
const start = Date.now();
|
|
128441
|
+
const rows = await getPrismaClient2().$queryRaw`
|
|
128442
|
+
SELECT id, content, type, level, importance, array_to_json(tags)::text AS tags,
|
|
128443
|
+
embedding, created_at, updated_at, access_count, user_id, session_id, project_id, agent_id
|
|
128444
|
+
FROM memories WHERE embedding IS NOT NULL ORDER BY created_at DESC LIMIT ${maxMemories}`;
|
|
128445
|
+
const items = [];
|
|
128446
|
+
for (const row of rows) {
|
|
128447
|
+
if (!row.embedding)
|
|
128448
|
+
continue;
|
|
128449
|
+
const buf = row.embedding instanceof Buffer ? row.embedding : Buffer.from(row.embedding);
|
|
128450
|
+
const vec = Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
|
|
128451
|
+
if (vec.every((v) => v === 0))
|
|
128452
|
+
continue;
|
|
128453
|
+
items.push({ row, vec });
|
|
128490
128454
|
}
|
|
128491
|
-
|
|
128492
|
-
|
|
128493
|
-
|
|
128494
|
-
|
|
128495
|
-
|
|
128496
|
-
|
|
128497
|
-
|
|
128498
|
-
|
|
128499
|
-
|
|
128500
|
-
|
|
128501
|
-
|
|
128502
|
-
|
|
128503
|
-
|
|
128504
|
-
|
|
128505
|
-
|
|
128506
|
-
|
|
128507
|
-
|
|
128508
|
-
|
|
128509
|
-
|
|
128510
|
-
|
|
128511
|
-
|
|
128512
|
-
|
|
128513
|
-
|
|
128514
|
-
|
|
128515
|
-
|
|
128516
|
-
|
|
128517
|
-
|
|
128518
|
-
|
|
128519
|
-
|
|
128520
|
-
|
|
128521
|
-
|
|
128522
|
-
|
|
128523
|
-
|
|
128524
|
-
|
|
128525
|
-
|
|
128526
|
-
|
|
128527
|
-
|
|
128528
|
-
|
|
128529
|
-
|
|
128530
|
-
|
|
128531
|
-
|
|
128532
|
-
|
|
128533
|
-
|
|
128534
|
-
|
|
128535
|
-
|
|
128536
|
-
|
|
128537
|
-
|
|
128538
|
-
|
|
128539
|
-
|
|
128540
|
-
|
|
128541
|
-
|
|
128542
|
-
|
|
128543
|
-
|
|
128544
|
-
|
|
128545
|
-
|
|
128546
|
-
|
|
128547
|
-
|
|
128548
|
-
|
|
128549
|
-
|
|
128550
|
-
|
|
128551
|
-
|
|
128552
|
-
|
|
128553
|
-
|
|
128554
|
-
|
|
128555
|
-
|
|
128556
|
-
|
|
128557
|
-
|
|
128558
|
-
|
|
128559
|
-
|
|
128560
|
-
|
|
128561
|
-
|
|
128562
|
-
|
|
128563
|
-
|
|
128564
|
-
|
|
128565
|
-
|
|
128566
|
-
|
|
128567
|
-
|
|
128568
|
-
|
|
128569
|
-
|
|
128570
|
-
|
|
128571
|
-
|
|
128572
|
-
|
|
128573
|
-
|
|
128574
|
-
|
|
128575
|
-
|
|
128576
|
-
|
|
128577
|
-
|
|
128578
|
-
taskId: state.taskId,
|
|
128579
|
-
type: checkpointType,
|
|
128580
|
-
compressedBytes: compressState(state).byteLength
|
|
128455
|
+
if (items.length < 3) {
|
|
128456
|
+
return { clusters: [], unclustered: items.length, durationMs: Date.now() - start };
|
|
128457
|
+
}
|
|
128458
|
+
const effectiveK = k ?? Math.max(2, Math.min(20, Math.round(Math.sqrt(items.length / 2))));
|
|
128459
|
+
const dim = items[0].vec.length;
|
|
128460
|
+
let centroids = this.kMeansPlusPlusInit(items.map((i) => i.vec), effectiveK);
|
|
128461
|
+
let assignments = new Int32Array(items.length);
|
|
128462
|
+
for (let iter = 0;iter < maxIter; iter++) {
|
|
128463
|
+
const newAssignments = new Int32Array(items.length);
|
|
128464
|
+
for (let i = 0;i < items.length; i++) {
|
|
128465
|
+
let bestDist = Infinity;
|
|
128466
|
+
let bestCluster = 0;
|
|
128467
|
+
for (let c = 0;c < centroids.length; c++) {
|
|
128468
|
+
const dist = this.euclideanDistanceSq(items[i].vec, centroids[c]);
|
|
128469
|
+
if (dist < bestDist) {
|
|
128470
|
+
bestDist = dist;
|
|
128471
|
+
bestCluster = c;
|
|
128472
|
+
}
|
|
128473
|
+
}
|
|
128474
|
+
newAssignments[i] = bestCluster;
|
|
128475
|
+
}
|
|
128476
|
+
let changed = false;
|
|
128477
|
+
for (let i = 0;i < items.length; i++) {
|
|
128478
|
+
if (newAssignments[i] !== assignments[i]) {
|
|
128479
|
+
changed = true;
|
|
128480
|
+
break;
|
|
128481
|
+
}
|
|
128482
|
+
}
|
|
128483
|
+
assignments = newAssignments;
|
|
128484
|
+
if (!changed)
|
|
128485
|
+
break;
|
|
128486
|
+
const newCentroids = [];
|
|
128487
|
+
const counts = [];
|
|
128488
|
+
for (let c = 0;c < centroids.length; c++) {
|
|
128489
|
+
newCentroids.push(Array.from({ length: dim }, () => 0));
|
|
128490
|
+
counts.push(0);
|
|
128491
|
+
}
|
|
128492
|
+
for (let i = 0;i < items.length; i++) {
|
|
128493
|
+
const c = assignments[i];
|
|
128494
|
+
counts[c]++;
|
|
128495
|
+
for (let d = 0;d < dim; d++) {
|
|
128496
|
+
newCentroids[c][d] += items[i].vec[d];
|
|
128497
|
+
}
|
|
128498
|
+
}
|
|
128499
|
+
for (let c = 0;c < centroids.length; c++) {
|
|
128500
|
+
if (counts[c] === 0)
|
|
128501
|
+
continue;
|
|
128502
|
+
for (let d = 0;d < dim; d++) {
|
|
128503
|
+
newCentroids[c][d] /= counts[c];
|
|
128504
|
+
}
|
|
128505
|
+
}
|
|
128506
|
+
centroids = newCentroids;
|
|
128507
|
+
}
|
|
128508
|
+
const clusterMap = new Map;
|
|
128509
|
+
for (let i = 0;i < items.length; i++) {
|
|
128510
|
+
const c = assignments[i];
|
|
128511
|
+
if (!clusterMap.has(c))
|
|
128512
|
+
clusterMap.set(c, []);
|
|
128513
|
+
clusterMap.get(c).push(items[i].row);
|
|
128514
|
+
}
|
|
128515
|
+
const clusters = [];
|
|
128516
|
+
let unclustered = 0;
|
|
128517
|
+
for (const [cIdx, members] of clusterMap) {
|
|
128518
|
+
if (members.length < 2) {
|
|
128519
|
+
unclustered += members.length;
|
|
128520
|
+
continue;
|
|
128521
|
+
}
|
|
128522
|
+
const label = this.generateLabel(members);
|
|
128523
|
+
const avgImportance = members.reduce((sum, m) => sum + m.importance, 0) / members.length;
|
|
128524
|
+
const totalAccess = members.reduce((sum, m) => sum + m.access_count, 0);
|
|
128525
|
+
const dominantType = this.getDominantType(members);
|
|
128526
|
+
clusters.push({
|
|
128527
|
+
id: `cluster_${Date.now()}_${cIdx}`,
|
|
128528
|
+
centroid: centroids[cIdx],
|
|
128529
|
+
memberIds: members.map((m) => m.id),
|
|
128530
|
+
label,
|
|
128531
|
+
importance: Math.round(avgImportance * 100) / 100,
|
|
128532
|
+
totalAccess,
|
|
128533
|
+
dominantType
|
|
128534
|
+
});
|
|
128535
|
+
}
|
|
128536
|
+
clusters.sort((a, b) => b.importance - a.importance);
|
|
128537
|
+
logger.info("MemoryClustering: complete", {
|
|
128538
|
+
inputMemories: items.length,
|
|
128539
|
+
clusters: clusters.length,
|
|
128540
|
+
unclustered,
|
|
128541
|
+
k: effectiveK
|
|
128581
128542
|
});
|
|
128582
|
-
return
|
|
128583
|
-
}
|
|
128584
|
-
getCheckpoint(checkpointId) {
|
|
128585
|
-
this.ensureHydrated();
|
|
128586
|
-
return this.mirror.get(checkpointId) ?? null;
|
|
128587
|
-
}
|
|
128588
|
-
listCheckpoints(options = {}) {
|
|
128589
|
-
this.ensureHydrated();
|
|
128590
|
-
return this.applyFilters(this.mirror.values(), options);
|
|
128543
|
+
return { clusters, unclustered, durationMs: Date.now() - start };
|
|
128591
128544
|
}
|
|
128592
|
-
|
|
128593
|
-
this.
|
|
128594
|
-
const
|
|
128595
|
-
|
|
128545
|
+
async findCluster(memoryId, cached2) {
|
|
128546
|
+
const result = cached2 ?? await this.clusterMemories();
|
|
128547
|
+
for (const cluster of result.clusters) {
|
|
128548
|
+
if (cluster.memberIds.includes(memoryId)) {
|
|
128549
|
+
return cluster;
|
|
128550
|
+
}
|
|
128551
|
+
}
|
|
128552
|
+
return null;
|
|
128596
128553
|
}
|
|
128597
|
-
|
|
128598
|
-
|
|
128599
|
-
|
|
128600
|
-
|
|
128554
|
+
async summarizeCluster(cluster) {
|
|
128555
|
+
const members = await getPrismaClient2().$queryRaw`SELECT content, type, importance FROM memories WHERE id = ANY(${cluster.memberIds}::text[]) ORDER BY importance DESC LIMIT 5`;
|
|
128556
|
+
if (members.length === 0)
|
|
128557
|
+
return cluster.label;
|
|
128558
|
+
const lead = members[0].content.split(/[.!?\n]/)[0].trim();
|
|
128559
|
+
const typeCount = members.length;
|
|
128560
|
+
const types6 = [...new Set(members.map((m) => m.type))].join(", ");
|
|
128561
|
+
return `[${cluster.label}] ${lead} (${typeCount} memories, types: ${types6})`;
|
|
128601
128562
|
}
|
|
128602
|
-
|
|
128603
|
-
|
|
128604
|
-
const
|
|
128605
|
-
|
|
128606
|
-
for (
|
|
128607
|
-
|
|
128608
|
-
|
|
128609
|
-
|
|
128563
|
+
kMeansPlusPlusInit(vectors, k) {
|
|
128564
|
+
const centroids = [];
|
|
128565
|
+
const firstIdx = Math.floor(Math.random() * vectors.length);
|
|
128566
|
+
centroids.push([...vectors[firstIdx]]);
|
|
128567
|
+
for (let c = 1;c < k; c++) {
|
|
128568
|
+
const distances = [];
|
|
128569
|
+
let totalDist = 0;
|
|
128570
|
+
for (const vec of vectors) {
|
|
128571
|
+
let minDist = Infinity;
|
|
128572
|
+
for (const centroid of centroids) {
|
|
128573
|
+
const d = this.euclideanDistanceSq(vec, centroid);
|
|
128574
|
+
if (d < minDist)
|
|
128575
|
+
minDist = d;
|
|
128576
|
+
}
|
|
128577
|
+
distances.push(minDist);
|
|
128578
|
+
totalDist += minDist;
|
|
128579
|
+
}
|
|
128580
|
+
if (totalDist === 0) {
|
|
128581
|
+
centroids.push([...vectors[Math.floor(Math.random() * vectors.length)]]);
|
|
128582
|
+
continue;
|
|
128583
|
+
}
|
|
128584
|
+
let r = Math.random() * totalDist;
|
|
128585
|
+
for (let i = 0;i < distances.length; i++) {
|
|
128586
|
+
r -= distances[i];
|
|
128587
|
+
if (r <= 0) {
|
|
128588
|
+
centroids.push([...vectors[i]]);
|
|
128589
|
+
break;
|
|
128610
128590
|
}
|
|
128611
128591
|
}
|
|
128592
|
+
if (centroids.length <= c) {
|
|
128593
|
+
centroids.push([...vectors[vectors.length - 1]]);
|
|
128594
|
+
}
|
|
128595
|
+
}
|
|
128596
|
+
return centroids;
|
|
128597
|
+
}
|
|
128598
|
+
generateLabel(members) {
|
|
128599
|
+
const stopWords = new Set([
|
|
128600
|
+
"the",
|
|
128601
|
+
"a",
|
|
128602
|
+
"an",
|
|
128603
|
+
"is",
|
|
128604
|
+
"are",
|
|
128605
|
+
"was",
|
|
128606
|
+
"were",
|
|
128607
|
+
"be",
|
|
128608
|
+
"been",
|
|
128609
|
+
"being",
|
|
128610
|
+
"have",
|
|
128611
|
+
"has",
|
|
128612
|
+
"had",
|
|
128613
|
+
"do",
|
|
128614
|
+
"does",
|
|
128615
|
+
"did",
|
|
128616
|
+
"will",
|
|
128617
|
+
"would",
|
|
128618
|
+
"could",
|
|
128619
|
+
"should",
|
|
128620
|
+
"may",
|
|
128621
|
+
"might",
|
|
128622
|
+
"shall",
|
|
128623
|
+
"can",
|
|
128624
|
+
"need",
|
|
128625
|
+
"dare",
|
|
128626
|
+
"ought",
|
|
128627
|
+
"used",
|
|
128628
|
+
"to",
|
|
128629
|
+
"of",
|
|
128630
|
+
"in",
|
|
128631
|
+
"for",
|
|
128632
|
+
"on",
|
|
128633
|
+
"with",
|
|
128634
|
+
"at",
|
|
128635
|
+
"by",
|
|
128636
|
+
"from",
|
|
128637
|
+
"as",
|
|
128638
|
+
"into",
|
|
128639
|
+
"through",
|
|
128640
|
+
"during",
|
|
128641
|
+
"before",
|
|
128642
|
+
"after",
|
|
128643
|
+
"above",
|
|
128644
|
+
"below",
|
|
128645
|
+
"between",
|
|
128646
|
+
"out",
|
|
128647
|
+
"off",
|
|
128648
|
+
"over",
|
|
128649
|
+
"under",
|
|
128650
|
+
"again",
|
|
128651
|
+
"further",
|
|
128652
|
+
"then",
|
|
128653
|
+
"once",
|
|
128654
|
+
"here",
|
|
128655
|
+
"there",
|
|
128656
|
+
"when",
|
|
128657
|
+
"where",
|
|
128658
|
+
"why",
|
|
128659
|
+
"how",
|
|
128660
|
+
"all",
|
|
128661
|
+
"both",
|
|
128662
|
+
"each",
|
|
128663
|
+
"few",
|
|
128664
|
+
"more",
|
|
128665
|
+
"most",
|
|
128666
|
+
"other",
|
|
128667
|
+
"some",
|
|
128668
|
+
"such",
|
|
128669
|
+
"no",
|
|
128670
|
+
"nor",
|
|
128671
|
+
"not",
|
|
128672
|
+
"only",
|
|
128673
|
+
"own",
|
|
128674
|
+
"same",
|
|
128675
|
+
"so",
|
|
128676
|
+
"than",
|
|
128677
|
+
"too",
|
|
128678
|
+
"very",
|
|
128679
|
+
"just",
|
|
128680
|
+
"don",
|
|
128681
|
+
"should",
|
|
128682
|
+
"now",
|
|
128683
|
+
"and",
|
|
128684
|
+
"but",
|
|
128685
|
+
"or",
|
|
128686
|
+
"if",
|
|
128687
|
+
"while",
|
|
128688
|
+
"that",
|
|
128689
|
+
"this",
|
|
128690
|
+
"it",
|
|
128691
|
+
"its",
|
|
128692
|
+
"what",
|
|
128693
|
+
"which",
|
|
128694
|
+
"who",
|
|
128695
|
+
"whom",
|
|
128696
|
+
"these",
|
|
128697
|
+
"those",
|
|
128698
|
+
"i",
|
|
128699
|
+
"me",
|
|
128700
|
+
"my",
|
|
128701
|
+
"myself",
|
|
128702
|
+
"we",
|
|
128703
|
+
"our",
|
|
128704
|
+
"ours",
|
|
128705
|
+
"you",
|
|
128706
|
+
"your",
|
|
128707
|
+
"he",
|
|
128708
|
+
"him",
|
|
128709
|
+
"his",
|
|
128710
|
+
"she",
|
|
128711
|
+
"her",
|
|
128712
|
+
"they",
|
|
128713
|
+
"them",
|
|
128714
|
+
"their",
|
|
128715
|
+
"de",
|
|
128716
|
+
"da",
|
|
128717
|
+
"do",
|
|
128718
|
+
"das",
|
|
128719
|
+
"dos",
|
|
128720
|
+
"em",
|
|
128721
|
+
"no",
|
|
128722
|
+
"na",
|
|
128723
|
+
"nos",
|
|
128724
|
+
"nas",
|
|
128725
|
+
"um",
|
|
128726
|
+
"uma",
|
|
128727
|
+
"uns",
|
|
128728
|
+
"umas",
|
|
128729
|
+
"para",
|
|
128730
|
+
"com",
|
|
128731
|
+
"por",
|
|
128732
|
+
"que",
|
|
128733
|
+
"se",
|
|
128734
|
+
"como",
|
|
128735
|
+
"mas",
|
|
128736
|
+
"ou",
|
|
128737
|
+
"quando",
|
|
128738
|
+
"mais",
|
|
128739
|
+
"tamb\xE9m",
|
|
128740
|
+
"j\xE1",
|
|
128741
|
+
"ainda",
|
|
128742
|
+
"sobre",
|
|
128743
|
+
"entre",
|
|
128744
|
+
"at\xE9",
|
|
128745
|
+
"sem",
|
|
128746
|
+
"sob",
|
|
128747
|
+
"esse",
|
|
128748
|
+
"essa",
|
|
128749
|
+
"este",
|
|
128750
|
+
"esta",
|
|
128751
|
+
"aquele",
|
|
128752
|
+
"aquela",
|
|
128753
|
+
"ele",
|
|
128754
|
+
"ela",
|
|
128755
|
+
"eles",
|
|
128756
|
+
"elas",
|
|
128757
|
+
"n\xF3s",
|
|
128758
|
+
"eu",
|
|
128759
|
+
"tu",
|
|
128760
|
+
"voc\xEA",
|
|
128761
|
+
"voc\xEAs",
|
|
128762
|
+
"meu",
|
|
128763
|
+
"minha",
|
|
128764
|
+
"seu",
|
|
128765
|
+
"sua"
|
|
128766
|
+
]);
|
|
128767
|
+
const wordFreq = new Map;
|
|
128768
|
+
for (const member of members) {
|
|
128769
|
+
const words = member.content.toLowerCase().replace(/[^a-z\u00E1\u00E0\u00E2\u00E3\u00E9\u00E8\u00EA\u00ED\u00EF\u00F3\u00F4\u00F5\u00FA\u00FC\u00E7\s-]/g, " ").split(/\s+/).filter((w) => w.length > 2 && !stopWords.has(w));
|
|
128770
|
+
const seen = new Set;
|
|
128771
|
+
for (const word of words) {
|
|
128772
|
+
if (seen.has(word))
|
|
128773
|
+
continue;
|
|
128774
|
+
seen.add(word);
|
|
128775
|
+
wordFreq.set(word, (wordFreq.get(word) ?? 0) + 1);
|
|
128776
|
+
}
|
|
128612
128777
|
}
|
|
128613
|
-
|
|
128614
|
-
|
|
128615
|
-
deleteCheckpoint(checkpointId) {
|
|
128616
|
-
const existed = this.mirror.has(checkpointId);
|
|
128617
|
-
this.mirror.delete(checkpointId);
|
|
128618
|
-
this.ensureHydrated();
|
|
128619
|
-
this.chainWrite(checkpointId, async () => {
|
|
128620
|
-
const prisma2 = this.getClient();
|
|
128621
|
-
await prisma2.$executeRaw`
|
|
128622
|
-
DELETE FROM task_checkpoints WHERE id = ${checkpointId}
|
|
128623
|
-
`;
|
|
128624
|
-
});
|
|
128625
|
-
return existed;
|
|
128778
|
+
const sorted = [...wordFreq.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(([word]) => word);
|
|
128779
|
+
return sorted.length > 0 ? sorted.join(", ") : "misc";
|
|
128626
128780
|
}
|
|
128627
|
-
|
|
128628
|
-
const
|
|
128629
|
-
|
|
128630
|
-
|
|
128631
|
-
for (const ckpt of this.mirror.values()) {
|
|
128632
|
-
if (ckpt.expiresAt != null && ckpt.expiresAt < now2) {
|
|
128633
|
-
toRemove.push(ckpt.id);
|
|
128634
|
-
count++;
|
|
128635
|
-
}
|
|
128636
|
-
}
|
|
128637
|
-
for (const id of toRemove)
|
|
128638
|
-
this.mirror.delete(id);
|
|
128639
|
-
if (count > 0) {
|
|
128640
|
-
this.chainWrite("__purge__", async () => {
|
|
128641
|
-
const prisma2 = this.getClient();
|
|
128642
|
-
await prisma2.$executeRaw`
|
|
128643
|
-
DELETE FROM task_checkpoints
|
|
128644
|
-
WHERE expires_at IS NOT NULL AND expires_at < ${now2}::bigint
|
|
128645
|
-
`;
|
|
128646
|
-
});
|
|
128647
|
-
logger.info("Expired checkpoints purged (PG)", { count });
|
|
128781
|
+
getDominantType(members) {
|
|
128782
|
+
const typeCounts = new Map;
|
|
128783
|
+
for (const m of members) {
|
|
128784
|
+
typeCounts.set(m.type, (typeCounts.get(m.type) ?? 0) + 1);
|
|
128648
128785
|
}
|
|
128649
|
-
|
|
128650
|
-
|
|
128651
|
-
|
|
128652
|
-
|
|
128653
|
-
|
|
128654
|
-
|
|
128655
|
-
const BATCH_SIZE = 1000;
|
|
128656
|
-
const existing = [];
|
|
128657
|
-
try {
|
|
128658
|
-
const prisma2 = this.getClient();
|
|
128659
|
-
for (let i = 0;i < memoryIds.length; i += BATCH_SIZE) {
|
|
128660
|
-
const batch = memoryIds.slice(i, i + BATCH_SIZE);
|
|
128661
|
-
const rows = await prisma2.$queryRaw`
|
|
128662
|
-
SELECT id FROM memories WHERE id IN (${import_prisma4.Prisma.join(batch)})
|
|
128663
|
-
`;
|
|
128664
|
-
for (const row of rows)
|
|
128665
|
-
existing.push(row.id);
|
|
128786
|
+
let dominant = "unknown";
|
|
128787
|
+
let maxCount = 0;
|
|
128788
|
+
for (const [type, count] of typeCounts) {
|
|
128789
|
+
if (count > maxCount) {
|
|
128790
|
+
maxCount = count;
|
|
128791
|
+
dominant = type;
|
|
128666
128792
|
}
|
|
128667
|
-
return existing;
|
|
128668
|
-
} catch (e) {
|
|
128669
|
-
logger.warn("countExistingMemoryIds failed (best-effort: assuming all exist)", {
|
|
128670
|
-
error: e.message
|
|
128671
|
-
});
|
|
128672
|
-
return memoryIds;
|
|
128673
|
-
}
|
|
128674
|
-
}
|
|
128675
|
-
getStats() {
|
|
128676
|
-
this.ensureHydrated();
|
|
128677
|
-
const checkpoints = Array.from(this.mirror.values());
|
|
128678
|
-
const byType = {};
|
|
128679
|
-
let totalSizeBytes = 0;
|
|
128680
|
-
let oldest;
|
|
128681
|
-
for (const c of checkpoints) {
|
|
128682
|
-
byType[c.checkpointType] = (byType[c.checkpointType] ?? 0) + 1;
|
|
128683
|
-
totalSizeBytes += compressState(c.state).byteLength;
|
|
128684
|
-
if (oldest == null || c.createdAt < oldest)
|
|
128685
|
-
oldest = c.createdAt;
|
|
128686
128793
|
}
|
|
128687
|
-
return
|
|
128688
|
-
totalCheckpoints: checkpoints.length,
|
|
128689
|
-
byType,
|
|
128690
|
-
totalSizeBytes,
|
|
128691
|
-
oldestCheckpointAge: oldest != null ? Date.now() - oldest : undefined
|
|
128692
|
-
};
|
|
128693
|
-
}
|
|
128694
|
-
ensureReady() {
|
|
128695
|
-
return this.ensureHydrated();
|
|
128794
|
+
return dominant;
|
|
128696
128795
|
}
|
|
128697
|
-
|
|
128698
|
-
|
|
128699
|
-
|
|
128700
|
-
|
|
128701
|
-
|
|
128702
|
-
for (const c of iter) {
|
|
128703
|
-
if (taskId && c.taskId !== taskId)
|
|
128704
|
-
continue;
|
|
128705
|
-
if (projectId && c.projectId !== projectId)
|
|
128706
|
-
continue;
|
|
128707
|
-
if (checkpointType && c.checkpointType !== checkpointType)
|
|
128708
|
-
continue;
|
|
128709
|
-
if (!includeExpired && c.expiresAt != null && c.expiresAt <= now2)
|
|
128710
|
-
continue;
|
|
128711
|
-
out.push(c);
|
|
128796
|
+
euclideanDistanceSq(a, b) {
|
|
128797
|
+
let sum = 0;
|
|
128798
|
+
for (let i = 0;i < a.length; i++) {
|
|
128799
|
+
const diff = a[i] - b[i];
|
|
128800
|
+
sum += diff * diff;
|
|
128712
128801
|
}
|
|
128713
|
-
|
|
128714
|
-
return out.slice(offset, offset + limit);
|
|
128715
|
-
}
|
|
128716
|
-
checkpointToMetadata(c) {
|
|
128717
|
-
return {
|
|
128718
|
-
id: c.id,
|
|
128719
|
-
taskId: c.taskId,
|
|
128720
|
-
taskDescription: c.taskDescription,
|
|
128721
|
-
agentId: c.agentId,
|
|
128722
|
-
projectId: c.projectId,
|
|
128723
|
-
checkpointType: c.checkpointType,
|
|
128724
|
-
parentCheckpointId: c.parentCheckpointId,
|
|
128725
|
-
createdAt: c.createdAt,
|
|
128726
|
-
expiresAt: c.expiresAt,
|
|
128727
|
-
compressedSizeBytes: compressState(c.state).byteLength,
|
|
128728
|
-
memoryCount: c.memoryIds.length,
|
|
128729
|
-
fileChangeCount: c.fileChanges.length
|
|
128730
|
-
};
|
|
128731
|
-
}
|
|
128732
|
-
chainWrite(key, fn) {
|
|
128733
|
-
const prev = this.inflight.get(key) ?? Promise.resolve();
|
|
128734
|
-
const next = prev.then(fn).catch((e) => {
|
|
128735
|
-
logger.warn("PgCheckpointStore write failed (best-effort)", {
|
|
128736
|
-
key,
|
|
128737
|
-
error: e.message
|
|
128738
|
-
});
|
|
128739
|
-
});
|
|
128740
|
-
this.inflight.set(key, next);
|
|
128741
|
-
next.then(() => {
|
|
128742
|
-
if (this.inflight.get(key) === next)
|
|
128743
|
-
this.inflight.delete(key);
|
|
128744
|
-
});
|
|
128745
|
-
}
|
|
128746
|
-
async __drain() {
|
|
128747
|
-
const pending = Array.from(this.inflight.values());
|
|
128748
|
-
if (pending.length > 0)
|
|
128749
|
-
await Promise.allSettled(pending);
|
|
128750
|
-
await new Promise((r) => setTimeout(r, 10));
|
|
128802
|
+
return sum;
|
|
128751
128803
|
}
|
|
128752
|
-
|
|
128753
|
-
|
|
128804
|
+
close() {
|
|
128805
|
+
MemoryClustering.instance = null;
|
|
128754
128806
|
}
|
|
128755
128807
|
}
|
|
128756
|
-
var
|
|
128757
|
-
var init_checkpoint_store_pg = __esm(() => {
|
|
128808
|
+
var init_memory_clustering = __esm(() => {
|
|
128758
128809
|
init_dist();
|
|
128759
128810
|
init_prisma_client();
|
|
128760
|
-
init_alias_resolver();
|
|
128761
|
-
init_schema_version();
|
|
128762
|
-
import_prisma4 = __toESM(require_prisma(), 1);
|
|
128763
|
-
});
|
|
128764
|
-
|
|
128765
|
-
// ../../packages/core/dist/services/checkpoint/checkpoint-manager.js
|
|
128766
|
-
var CheckpointManager;
|
|
128767
|
-
var init_checkpoint_manager = __esm(() => {
|
|
128768
|
-
init_config();
|
|
128769
|
-
init_checkpoint_store_pg();
|
|
128770
|
-
CheckpointManager = class CheckpointManager extends PgCheckpointStore {
|
|
128771
|
-
static instance = null;
|
|
128772
|
-
static getInstance() {
|
|
128773
|
-
requirePostgresDatabaseUrl();
|
|
128774
|
-
return this.instance ??= new CheckpointManager;
|
|
128775
|
-
}
|
|
128776
|
-
async restoreCheckpoint(checkpointId) {
|
|
128777
|
-
const checkpoint = this.getCheckpoint(checkpointId);
|
|
128778
|
-
if (!checkpoint)
|
|
128779
|
-
return null;
|
|
128780
|
-
const existing = new Set(await this.countExistingMemoryIds(checkpoint.memoryIds));
|
|
128781
|
-
const validMemoryIds = checkpoint.memoryIds.filter((id) => existing.has(id));
|
|
128782
|
-
const missingMemoryIds = checkpoint.memoryIds.filter((id) => !existing.has(id));
|
|
128783
|
-
const fileConflicts = [];
|
|
128784
|
-
const restoreInstructions = [
|
|
128785
|
-
`Restore checkpoint ${checkpoint.id} for task ${checkpoint.taskId}.`,
|
|
128786
|
-
missingMemoryIds.length ? `Missing memories: ${missingMemoryIds.join(", ")}.` : "All referenced memories are available."
|
|
128787
|
-
].join(`
|
|
128788
|
-
`);
|
|
128789
|
-
return { checkpoint, validMemoryIds, missingMemoryIds, fileConflicts, restoreInstructions };
|
|
128790
|
-
}
|
|
128791
|
-
};
|
|
128792
128811
|
});
|
|
128793
128812
|
|
|
128794
128813
|
// ../../packages/core/dist/services/checkpoint/auto-checkpointer.js
|