@massa-ai/mcp-client 1.37.0 → 1.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -108231,6 +108231,7 @@ var init_registry = __esm(() => {
108231
108231
  search_analytics: { storeId: "search_analytics", identityColumn: "project_id", mutable: true },
108232
108232
  search_events: { storeId: "search_events", identityColumn: "project_id", mutable: true },
108233
108233
  synapse_sessions: { storeId: "synapse_sessions", identityColumn: "workspace_id", mutable: true },
108234
+ managed_runs: { storeId: "managed_runs", identityColumn: "project_id", mutable: true },
108234
108235
  operation_log: { storeId: "operation_log", identityColumn: "project_id", mutable: false },
108235
108236
  project_identity_operations: {
108236
108237
  storeId: "project_identity_operations",
@@ -127386,1409 +127387,1427 @@ var init_observation_consolidation_job = __esm(() => {
127386
127387
  observationConsolidationJob = new ObservationConsolidationJob;
127387
127388
  });
127388
127389
 
127389
- // ../../packages/core/dist/services/scheduler/scheduler-defaults.js
127390
- function envBool2(key, fallback) {
127391
- const raw2 = process.env[key];
127392
- if (raw2 === undefined)
127393
- return fallback;
127394
- return raw2 === "true" || raw2 === "1";
127395
- }
127396
- function envNum2(key, fallback) {
127397
- const raw2 = process.env[key];
127398
- if (raw2 === undefined || raw2 === "")
127399
- return fallback;
127400
- const n = Number(raw2);
127401
- return Number.isFinite(n) && n > 0 ? n : fallback;
127390
+ // ../../packages/core/dist/services/checkpoint/checkpoint-store-pg.js
127391
+ function toNum4(v) {
127392
+ if (v == null)
127393
+ return null;
127394
+ return typeof v === "bigint" ? Number(v) : v;
127402
127395
  }
127403
- function applySafeDefaults(job) {
127404
- if (!envBool2("MASSA_AI_SCHEDULER_SAFE_DEFAULTS", false)) {
127405
- return job;
127406
- }
127407
- if (job.jobKind === "memory-consolidation") {
127408
- const currentInterval = job.schedule.intervalMs ?? THIRTY_MIN;
127409
- const safeInterval = Math.max(currentInterval, THIRTY_MIN);
127410
- return {
127411
- ...job,
127412
- defaultEnabled: true,
127413
- schedule: { type: "interval", intervalMs: safeInterval }
127414
- };
127415
- }
127416
- if (job.jobKind === "decay-sweep") {
127417
- const currentInterval = job.schedule.intervalMs ?? ONE_HOUR;
127418
- const safeInterval = Math.max(currentInterval, ONE_HOUR);
127419
- return {
127420
- ...job,
127421
- defaultEnabled: true,
127422
- schedule: { type: "interval", intervalMs: safeInterval }
127423
- };
127424
- }
127425
- return job;
127396
+ function compressState(state) {
127397
+ const json3 = JSON.stringify(state);
127398
+ return Buffer.from(Bun.deflateSync(Buffer.from(json3, "utf-8")));
127426
127399
  }
127427
- function registerDefaultJobs(scheduler) {
127428
- scheduler.registerHandler("memory-consolidation", async () => {
127429
- const { memoryConsolidationJob: memoryConsolidationJob2 } = await Promise.resolve().then(() => (init_memory_consolidation_job(), exports_memory_consolidation_job));
127430
- await memoryConsolidationJob2.consolidate();
127431
- });
127432
- scheduler.registerHandler("decay-sweep", async () => {
127433
- const { memoryConsolidationJob: memoryConsolidationJob2 } = await Promise.resolve().then(() => (init_memory_consolidation_job(), exports_memory_consolidation_job));
127434
- await memoryConsolidationJob2.consolidate();
127435
- });
127436
- scheduler.registerHandler("auto-improve", async (job) => {
127437
- const { autoImproveJob: autoImproveJob2 } = await Promise.resolve().then(() => (init_auto_improve_job(), exports_auto_improve_job));
127438
- const projectId = job.payload?.projectId ?? "default";
127439
- await autoImproveJob2.runOnce(projectId);
127440
- });
127441
- scheduler.registerHandler("observation-bridge", async (job) => {
127442
- const { observationConsolidationJob: observationConsolidationJob2 } = await Promise.resolve().then(() => (init_observation_consolidation_job(), exports_observation_consolidation_job));
127443
- const projectId = job.payload?.projectId ?? "default";
127444
- await observationConsolidationJob2.runOnce(projectId);
127445
- });
127446
- for (const rawDef of DEFAULT_SCHEDULED_JOBS) {
127447
- const def = applySafeDefaults(rawDef);
127448
- const enabled = envBool2(def.enableEnvVar, def.defaultEnabled);
127449
- const intervalMs = envNum2(def.intervalEnvVar, def.schedule.intervalMs ?? THIRTY_MIN);
127450
- const schedule = { type: "interval", intervalMs };
127451
- scheduler.registerOrResumeJob({
127452
- id: def.id,
127453
- name: def.name,
127454
- jobKind: def.jobKind,
127455
- schedule,
127456
- nextRunAt: 0,
127457
- enabled
127458
- });
127459
- }
127460
- logger.info("Scheduler default jobs registered", {
127461
- handlers: scheduler.registeredKinds(),
127462
- jobs: DEFAULT_SCHEDULED_JOBS.length
127463
- });
127400
+ function decompressState(data) {
127401
+ const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
127402
+ const inflated = Bun.inflateSync(new Uint8Array(buf));
127403
+ return JSON.parse(Buffer.from(inflated).toString("utf-8"));
127464
127404
  }
127465
- var MIN, THIRTY_MIN, ONE_HOUR, DEFAULT_SCHEDULED_JOBS;
127466
- var init_scheduler_defaults = __esm(() => {
127467
- init_dist();
127468
- MIN = 60 * 1000;
127469
- THIRTY_MIN = 30 * MIN;
127470
- ONE_HOUR = 60 * MIN;
127471
- DEFAULT_SCHEDULED_JOBS = [
127472
- {
127473
- id: "scheduled-memory-consolidation",
127474
- name: "Memory Consolidation (clock)",
127475
- jobKind: "memory-consolidation",
127476
- schedule: { type: "interval", intervalMs: THIRTY_MIN },
127477
- defaultEnabled: false,
127478
- enableEnvVar: "MASSA_AI_SCHEDULER_CONSOLIDATION_ENABLED",
127479
- intervalEnvVar: "MASSA_AI_SCHEDULER_CONSOLIDATION_INTERVAL_MS"
127480
- },
127481
- {
127482
- id: "scheduled-decay-sweep",
127483
- name: "Decay Sweep (clock)",
127484
- jobKind: "decay-sweep",
127485
- schedule: { type: "interval", intervalMs: ONE_HOUR },
127486
- defaultEnabled: false,
127487
- enableEnvVar: "MASSA_AI_SCHEDULER_DECAY_ENABLED",
127488
- intervalEnvVar: "MASSA_AI_SCHEDULER_DECAY_INTERVAL_MS"
127489
- },
127490
- {
127491
- id: "scheduled-auto-improve",
127492
- name: "Auto-Improve (clock)",
127493
- jobKind: "auto-improve",
127494
- schedule: { type: "interval", intervalMs: THIRTY_MIN },
127495
- defaultEnabled: false,
127496
- enableEnvVar: "MASSA_AI_SCHEDULER_AUTO_IMPROVE_ENABLED",
127497
- intervalEnvVar: "MASSA_AI_SCHEDULER_AUTO_IMPROVE_INTERVAL_MS"
127498
- },
127499
- {
127500
- id: "scheduled-observation-bridge",
127501
- name: "Observation Bridge (clock)",
127502
- jobKind: "observation-bridge",
127503
- schedule: { type: "interval", intervalMs: THIRTY_MIN },
127504
- defaultEnabled: false,
127505
- enableEnvVar: "MASSA_AI_SCHEDULER_OBSERVATION_BRIDGE_ENABLED",
127506
- intervalEnvVar: "MASSA_AI_SCHEDULER_OBSERVATION_BRIDGE_INTERVAL_MS"
127507
- }
127508
- ];
127509
- });
127510
-
127511
- // ../../packages/core/dist/services/scheduler/index.js
127512
- var init_scheduler2 = __esm(() => {
127513
- init_scheduler();
127514
- init_scheduler_store_pg();
127515
- init_scheduler_store_factory();
127516
- init_scheduler_cron();
127517
- init_scheduler_defaults();
127518
- });
127519
127405
 
127520
- // ../../packages/core/dist/services/pricing/models-dev-client.js
127521
- import fs11 from "fs/promises";
127522
- import { existsSync as existsSync4 } from "fs";
127523
- import path16 from "path";
127524
- function getModelsDevClient() {
127525
- if (!clientInstance) {
127526
- clientInstance = new ModelsDevClient;
127406
+ class PgCheckpointStore {
127407
+ prisma;
127408
+ mirror = new Map;
127409
+ hydrated = false;
127410
+ hydrating = null;
127411
+ hydrateFailedAt = 0;
127412
+ static HYDRATE_RETRY_MS = 30000;
127413
+ inflight = new Map;
127414
+ getClient() {
127415
+ if (!this.prisma)
127416
+ this.prisma = getPrismaClient2();
127417
+ return this.prisma;
127527
127418
  }
127528
- return clientInstance;
127529
- }
127530
- var ModelsDevClient, clientInstance = null;
127531
- var init_models_dev_client = __esm(() => {
127532
- init_dist();
127533
- init_dist();
127534
- ModelsDevClient = class ModelsDevClient {
127535
- static API_URL = "https://models.dev/api.json";
127536
- static MEMORY_CACHE_TTL = 3600 * 1000;
127537
- static LOCAL_CACHE_TTL = 24 * 3600 * 1000;
127538
- static LOCAL_CACHE_FILE = "pricing-cache.json";
127539
- memoryCache = null;
127540
- memoryCacheTimestamp = 0;
127541
- getLocalCachePath() {
127542
- const dataDir = config.get("dataDir");
127543
- return path16.join(dataDir, ModelsDevClient.LOCAL_CACHE_FILE);
127419
+ ensureHydrated() {
127420
+ if (this.hydrated)
127421
+ return Promise.resolve();
127422
+ if (this.hydrating)
127423
+ return this.hydrating;
127424
+ if (this.hydrateFailedAt > 0 && Date.now() - this.hydrateFailedAt < PgCheckpointStore.HYDRATE_RETRY_MS) {
127425
+ return Promise.resolve();
127544
127426
  }
127545
- async loadLocalCache() {
127546
- const cachePath = this.getLocalCachePath();
127427
+ this.hydrating = (async () => {
127547
127428
  try {
127548
- if (!existsSync4(cachePath)) {
127549
- return null;
127550
- }
127551
- const content = await fs11.readFile(cachePath, "utf-8");
127552
- const data = JSON.parse(content);
127553
- const age = Date.now() - data.timestamp;
127554
- if (age > ModelsDevClient.LOCAL_CACHE_TTL) {
127555
- logger.debug("Local pricing cache expired", {
127556
- ageHours: Math.round(age / 3600000)
127557
- });
127429
+ const prisma2 = this.getClient();
127430
+ const rows = await prisma2.$queryRaw`
127431
+ SELECT * FROM task_checkpoints
127432
+ `;
127433
+ const next = new Map;
127434
+ const dbIds = new Set;
127435
+ for (const row of rows) {
127436
+ dbIds.add(row.id);
127437
+ next.set(row.id, this.rowToCheckpoint(row));
127558
127438
  }
127559
- const models = new Map;
127560
- for (const [key, value] of Object.entries(data.models)) {
127561
- models.set(key, value);
127439
+ for (const [id, existing] of this.mirror) {
127440
+ if (!dbIds.has(id))
127441
+ next.set(id, existing);
127562
127442
  }
127563
- logger.debug("Loaded pricing from local cache", {
127564
- models: models.size,
127565
- ageHours: Math.round(age / 3600000)
127443
+ this.mirror = next;
127444
+ this.hydrated = true;
127445
+ this.hydrateFailedAt = 0;
127446
+ logger.info("PgCheckpointStore hydrated", {
127447
+ rows: this.mirror.size
127566
127448
  });
127567
- return models;
127568
- } catch (error51) {
127569
- logger.debug("Failed to load local pricing cache", {
127570
- error: error51.message
127449
+ } catch (e) {
127450
+ this.hydrateFailedAt = Date.now();
127451
+ logger.warn("PgCheckpointStore hydrate failed (best-effort)", {
127452
+ error: e.message
127571
127453
  });
127572
- return null;
127454
+ } finally {
127455
+ this.hydrating = null;
127573
127456
  }
127457
+ })();
127458
+ return this.hydrating;
127459
+ }
127460
+ rowToCheckpoint(row) {
127461
+ const state = decompressState(row.state);
127462
+ const storedSchemaVersion = toNum4(row.state_schema_version);
127463
+ if (storedSchemaVersion != null) {
127464
+ const storedVersionString = String(storedSchemaVersion);
127465
+ const normalized = /^\d+$/u.test(storedVersionString) ? `${storedVersionString}.0.0` : storedVersionString;
127466
+ assertSchemaSupported("checkpoint", normalized, SUPPORTED_CHECKPOINT_STATE_SCHEMA_VERSION);
127574
127467
  }
127575
- async saveLocalCache(models) {
127576
- const cachePath = this.getLocalCachePath();
127577
- try {
127578
- const dir = path16.dirname(cachePath);
127579
- await fs11.mkdir(dir, { recursive: true });
127580
- const data = {
127581
- timestamp: Date.now(),
127582
- version: "1.0.0",
127583
- models: Object.fromEntries(models)
127584
- };
127585
- await fs11.writeFile(cachePath, JSON.stringify(data), "utf-8");
127586
- logger.debug("Saved pricing to local cache", {
127587
- models: models.size,
127588
- path: cachePath
127589
- });
127590
- } catch (error51) {
127591
- logger.warn("Failed to save local pricing cache", {
127592
- error: error51.message
127593
- });
127594
- }
127595
- }
127596
- getBundledDefaults() {
127597
- const models = new Map;
127598
- const defaults3 = [
127599
- {
127600
- id: "ollama/nomic-embed-text",
127601
- name: "Nomic Embed Text",
127602
- provider: "Ollama",
127603
- family: "nomic",
127604
- inputCostPerMillion: 0,
127605
- outputCostPerMillion: 0,
127606
- contextWindow: 8192
127607
- },
127608
- {
127609
- id: "ollama/all-minilm",
127610
- name: "All MiniLM",
127611
- provider: "Ollama",
127612
- family: "minilm",
127613
- inputCostPerMillion: 0,
127614
- outputCostPerMillion: 0,
127615
- contextWindow: 512
127616
- },
127617
- {
127618
- id: "ollama/mxbai-embed-large",
127619
- name: "MxBai Embed Large",
127620
- provider: "Ollama",
127621
- family: "mxbai",
127622
- inputCostPerMillion: 0,
127623
- outputCostPerMillion: 0,
127624
- contextWindow: 512
127625
- },
127626
- {
127627
- id: "mistral-embed",
127628
- name: "Mistral Embed",
127629
- provider: "Mistral",
127630
- family: "mistral",
127631
- inputCostPerMillion: 0.1,
127632
- outputCostPerMillion: 0,
127633
- contextWindow: 8192
127634
- },
127635
- {
127636
- id: "codestral-embed",
127637
- name: "Codestral Embed",
127638
- provider: "Mistral",
127639
- family: "codestral",
127640
- inputCostPerMillion: 0.1,
127641
- outputCostPerMillion: 0,
127642
- contextWindow: 32768
127643
- },
127644
- {
127645
- id: "mistral-small-latest",
127646
- name: "Mistral Small",
127647
- provider: "Mistral",
127648
- family: "mistral",
127649
- inputCostPerMillion: 0.2,
127650
- outputCostPerMillion: 0.6,
127651
- contextWindow: 32768,
127652
- maxOutputTokens: 8192
127653
- },
127654
- {
127655
- id: "text-embedding-3-small",
127656
- name: "Text Embedding 3 Small",
127657
- provider: "OpenAI",
127658
- family: "embedding",
127659
- inputCostPerMillion: 0.02,
127660
- outputCostPerMillion: 0,
127661
- contextWindow: 8191
127662
- },
127663
- {
127664
- id: "gpt-4o-mini",
127665
- name: "GPT-4o Mini",
127666
- provider: "OpenAI",
127667
- family: "gpt-4",
127668
- inputCostPerMillion: 0.15,
127669
- outputCostPerMillion: 0.6,
127670
- contextWindow: 128000,
127671
- maxOutputTokens: 16384
127672
- },
127673
- {
127674
- id: "claude-3-haiku-20240307",
127675
- name: "Claude 3 Haiku",
127676
- provider: "Anthropic",
127677
- family: "claude-3",
127678
- inputCostPerMillion: 0.25,
127679
- outputCostPerMillion: 1.25,
127680
- contextWindow: 200000,
127681
- maxOutputTokens: 4096
127682
- },
127683
- {
127684
- id: "claude-3-5-sonnet-20241022",
127685
- name: "Claude 3.5 Sonnet",
127686
- provider: "Anthropic",
127687
- family: "claude-3.5",
127688
- inputCostPerMillion: 3,
127689
- outputCostPerMillion: 15,
127690
- contextWindow: 200000,
127691
- maxOutputTokens: 8192
127692
- },
127693
- {
127694
- id: "gemini-1.5-flash",
127695
- name: "Gemini 1.5 Flash",
127696
- provider: "Google",
127697
- family: "gemini",
127698
- inputCostPerMillion: 0.075,
127699
- outputCostPerMillion: 0.3,
127700
- contextWindow: 1e6,
127701
- maxOutputTokens: 8192
127702
- },
127703
- {
127704
- id: "embed-english-v3.0",
127705
- name: "Embed English v3",
127706
- provider: "Cohere",
127707
- family: "embed",
127708
- inputCostPerMillion: 0.1,
127709
- outputCostPerMillion: 0,
127710
- contextWindow: 512
127711
- }
127712
- ];
127713
- for (const pricing of defaults3) {
127714
- models.set(pricing.id, pricing);
127715
- }
127716
- logger.info("Using bundled pricing defaults (offline mode)", {
127717
- models: models.size
127718
- });
127719
- return models;
127720
- }
127721
- async fetchAllModels() {
127722
- if (this.memoryCache && Date.now() - this.memoryCacheTimestamp < ModelsDevClient.MEMORY_CACHE_TTL) {
127723
- logger.debug("Using in-memory pricing cache");
127724
- return this.memoryCache;
127725
- }
127726
- const localCache = await this.loadLocalCache();
127727
- let remoteModels = null;
127728
- try {
127729
- remoteModels = await this.fetchRemote();
127730
- await this.saveLocalCache(remoteModels);
127731
- } catch (error51) {
127732
- logger.debug("Remote pricing fetch failed (offline?)", {
127733
- error: error51.message
127734
- });
127735
- }
127736
- let result;
127737
- if (remoteModels) {
127738
- result = remoteModels;
127739
- logger.debug("Using remote pricing data");
127740
- } else if (localCache) {
127741
- result = localCache;
127742
- logger.info("Using local pricing cache (offline mode)");
127743
- } else {
127744
- result = this.getBundledDefaults();
127745
- }
127746
- this.memoryCache = result;
127747
- this.memoryCacheTimestamp = Date.now();
127748
- return result;
127749
- }
127750
- async fetchRemote() {
127751
- const controller = new AbortController;
127752
- const timeout = setTimeout(() => controller.abort(), 1e4);
127753
- try {
127754
- logger.info("Fetching pricing data from models.dev API");
127755
- const response = await fetch(ModelsDevClient.API_URL, {
127756
- signal: controller.signal
127757
- });
127758
- if (!response.ok) {
127759
- throw new Error(`HTTP ${response.status}: ${response.statusText}`);
127760
- }
127761
- const data = await response.json();
127762
- const models = new Map;
127763
- let totalModels = 0;
127764
- let modelsWithPricing = 0;
127765
- for (const [providerId, provider] of Object.entries(data)) {
127766
- for (const [modelId, model] of Object.entries(provider.models || {})) {
127767
- totalModels++;
127768
- const cost = model.cost || { input: 0, output: 0 };
127769
- if (cost.input === 0 && cost.output === 0) {
127770
- continue;
127771
- }
127772
- modelsWithPricing++;
127773
- const normalizedId = this.normalizeModelId(modelId);
127774
- const pricing = {
127775
- id: normalizedId,
127776
- name: model.name || modelId,
127777
- provider: provider.name || providerId,
127778
- family: model.family,
127779
- inputCostPerMillion: cost.input,
127780
- outputCostPerMillion: cost.output,
127781
- cacheReadCostPerMillion: cost.cache_read,
127782
- contextWindow: model.limit?.context,
127783
- maxOutputTokens: model.limit?.output,
127784
- lastUpdated: model.last_updated
127785
- };
127786
- models.set(modelId, pricing);
127787
- if (normalizedId !== modelId) {
127788
- models.set(normalizedId, pricing);
127789
- }
127790
- this.addCommonAliases(models, modelId, pricing);
127791
- }
127792
- }
127793
- logger.info(`Loaded ${modelsWithPricing} models with pricing (out of ${totalModels} total)`);
127794
- return models;
127795
- } finally {
127796
- clearTimeout(timeout);
127797
- }
127798
- }
127799
- async getModelPricing(modelId) {
127800
- const models = await this.fetchAllModels();
127801
- let pricing = models.get(modelId);
127802
- if (pricing)
127803
- return pricing;
127804
- const normalized = this.normalizeModelId(modelId);
127805
- pricing = models.get(normalized);
127806
- if (pricing)
127807
- return pricing;
127808
- const lowerModelId = modelId.toLowerCase();
127809
- for (const [key, value] of models.entries()) {
127810
- if (key.toLowerCase() === lowerModelId) {
127811
- return value;
127812
- }
127813
- }
127814
- logger.warn(`Model pricing not found: ${modelId}`);
127815
- return null;
127816
- }
127817
- async searchModels(query) {
127818
- const models = await this.fetchAllModels();
127819
- const lowerQuery = query.toLowerCase();
127820
- const results = [];
127821
- const seen = new Set;
127822
- for (const pricing of models.values()) {
127823
- const key = `${pricing.provider}:${pricing.name}`;
127824
- if (seen.has(key))
127825
- continue;
127826
- seen.add(key);
127827
- if (pricing.id.toLowerCase().includes(lowerQuery) || pricing.name.toLowerCase().includes(lowerQuery) || pricing.provider.toLowerCase().includes(lowerQuery) || pricing.family && pricing.family.toLowerCase().includes(lowerQuery)) {
127828
- results.push(pricing);
127829
- }
127830
- }
127831
- results.sort((a, b) => b.inputCostPerMillion - a.inputCostPerMillion);
127832
- return results;
127833
- }
127834
- async getTopExpensiveModels(limit = 10) {
127835
- const models = await this.fetchAllModels();
127836
- const unique = new Map;
127837
- for (const pricing of models.values()) {
127838
- const key = `${pricing.provider}:${pricing.name}`;
127839
- if (!unique.has(key)) {
127840
- unique.set(key, pricing);
127841
- }
127842
- }
127843
- return Array.from(unique.values()).sort((a, b) => b.inputCostPerMillion - a.inputCostPerMillion).slice(0, limit);
127844
- }
127845
- async getStatistics() {
127846
- const models = await this.fetchAllModels();
127847
- const unique = new Map;
127848
- const providers = new Set;
127849
- let totalInputCost = 0;
127850
- let totalOutputCost = 0;
127851
- for (const pricing of models.values()) {
127852
- const key = `${pricing.provider}:${pricing.name}`;
127853
- if (!unique.has(key)) {
127854
- unique.set(key, pricing);
127855
- providers.add(pricing.provider);
127856
- totalInputCost += pricing.inputCostPerMillion;
127857
- totalOutputCost += pricing.outputCostPerMillion;
127468
+ return {
127469
+ id: row.id,
127470
+ taskId: row.task_id,
127471
+ taskDescription: row.task_description ?? undefined,
127472
+ agentId: row.agent_id ?? undefined,
127473
+ projectId: row.project_id ?? undefined,
127474
+ state,
127475
+ memoryIds: row.memory_ids ? JSON.parse(row.memory_ids) : [],
127476
+ fileChanges: row.file_changes ? JSON.parse(row.file_changes) : [],
127477
+ checkpointType: row.checkpoint_type,
127478
+ parentCheckpointId: row.parent_checkpoint_id ?? undefined,
127479
+ createdAt: toNum4(row.created_at) ?? Date.now(),
127480
+ expiresAt: toNum4(row.expires_at) ?? undefined
127481
+ };
127482
+ }
127483
+ createCheckpoint(state, options = {}) {
127484
+ const {
127485
+ agentId,
127486
+ projectId,
127487
+ checkpointType = CheckpointType.MANUAL,
127488
+ memoryIds = [],
127489
+ fileChanges = [],
127490
+ parentCheckpointId,
127491
+ ttlMs = 7 * 24 * 60 * 60 * 1000
127492
+ } = options;
127493
+ const id = `ckpt_${checkpointType}_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`;
127494
+ const now2 = Date.now();
127495
+ const expiresAt = now2 + ttlMs;
127496
+ const checkpoint = {
127497
+ id,
127498
+ taskId: state.taskId,
127499
+ taskDescription: state.description,
127500
+ agentId,
127501
+ projectId,
127502
+ state,
127503
+ memoryIds,
127504
+ fileChanges,
127505
+ checkpointType,
127506
+ parentCheckpointId,
127507
+ createdAt: now2,
127508
+ expiresAt
127509
+ };
127510
+ this.mirror.set(id, checkpoint);
127511
+ this.ensureHydrated();
127512
+ this.chainWrite(id, async () => {
127513
+ const prisma2 = this.getClient();
127514
+ const canonicalProjectId = projectId ? await getProjectIdentityAliasResolver().resolve(projectId) : projectId;
127515
+ const compressed = compressState(state);
127516
+ await prisma2.$executeRaw`
127517
+ INSERT INTO task_checkpoints (
127518
+ id, task_id, task_description, agent_id, project_id,
127519
+ state, state_schema_version,
127520
+ memory_ids, file_changes,
127521
+ checkpoint_type, parent_checkpoint_id,
127522
+ created_at, expires_at
127523
+ ) VALUES (
127524
+ ${id},
127525
+ ${state.taskId},
127526
+ ${state.description ?? null},
127527
+ ${agentId ?? null},
127528
+ ${canonicalProjectId ?? null},
127529
+ ${compressed},
127530
+ 1,
127531
+ ${JSON.stringify(memoryIds)},
127532
+ ${JSON.stringify(fileChanges)},
127533
+ ${checkpointType},
127534
+ ${parentCheckpointId ?? null},
127535
+ ${now2}::bigint,
127536
+ ${expiresAt}::bigint
127537
+ )
127538
+ ON CONFLICT (id) DO UPDATE SET
127539
+ task_id = EXCLUDED.task_id,
127540
+ task_description = EXCLUDED.task_description,
127541
+ agent_id = EXCLUDED.agent_id,
127542
+ project_id = EXCLUDED.project_id,
127543
+ state = EXCLUDED.state,
127544
+ state_schema_version = EXCLUDED.state_schema_version,
127545
+ memory_ids = EXCLUDED.memory_ids,
127546
+ file_changes = EXCLUDED.file_changes,
127547
+ checkpoint_type = EXCLUDED.checkpoint_type,
127548
+ parent_checkpoint_id = EXCLUDED.parent_checkpoint_id,
127549
+ created_at = EXCLUDED.created_at,
127550
+ expires_at = EXCLUDED.expires_at
127551
+ `;
127552
+ });
127553
+ logger.info("Checkpoint created (PG)", {
127554
+ id,
127555
+ taskId: state.taskId,
127556
+ type: checkpointType,
127557
+ compressedBytes: compressState(state).byteLength
127558
+ });
127559
+ return checkpoint;
127560
+ }
127561
+ getCheckpoint(checkpointId) {
127562
+ this.ensureHydrated();
127563
+ return this.mirror.get(checkpointId) ?? null;
127564
+ }
127565
+ listCheckpoints(options = {}) {
127566
+ this.ensureHydrated();
127567
+ return this.applyFilters(this.mirror.values(), options);
127568
+ }
127569
+ listCheckpointsMetadata(options = {}) {
127570
+ this.ensureHydrated();
127571
+ const filtered = this.applyFilters(this.mirror.values(), options);
127572
+ return filtered.map((c) => this.checkpointToMetadata(c));
127573
+ }
127574
+ getCheckpointState(checkpointId) {
127575
+ this.ensureHydrated();
127576
+ const ckpt = this.mirror.get(checkpointId);
127577
+ return ckpt ? ckpt.state : null;
127578
+ }
127579
+ getLatestCheckpoint(taskId) {
127580
+ this.ensureHydrated();
127581
+ const now2 = Date.now();
127582
+ let latest = null;
127583
+ for (const ckpt of this.mirror.values()) {
127584
+ if (ckpt.taskId === taskId && (ckpt.expiresAt == null || ckpt.expiresAt > now2)) {
127585
+ if (!latest || ckpt.createdAt > latest.createdAt) {
127586
+ latest = ckpt;
127858
127587
  }
127859
127588
  }
127860
- const modelList = Array.from(unique.values());
127861
- modelList.sort((a, b) => a.inputCostPerMillion - b.inputCostPerMillion);
127862
- return {
127863
- totalModels: modelList.length,
127864
- totalProviders: providers.size,
127865
- avgInputCost: modelList.length > 0 ? totalInputCost / modelList.length : 0,
127866
- avgOutputCost: modelList.length > 0 ? totalOutputCost / modelList.length : 0,
127867
- cheapestModel: modelList[0] || null,
127868
- mostExpensiveModel: modelList[modelList.length - 1] || null
127869
- };
127870
- }
127871
- normalizeModelId(modelId) {
127872
- let normalized = modelId.replace(/^(openai|anthropic|google|meta|mistral|cohere)\//i, "").replace(/^(claude-|gpt-|gemini-|llama-)/i, (match2) => match2.toLowerCase());
127873
- return normalized;
127874
127589
  }
127875
- addCommonAliases(models, modelId, pricing) {
127876
- if (modelId.includes("gpt-4") && !modelId.includes("turbo")) {
127877
- models.set("gpt-4", pricing);
127878
- }
127879
- if (modelId.includes("gpt-4-turbo")) {
127880
- models.set("gpt-4-turbo", pricing);
127881
- }
127882
- if (modelId.includes("gpt-3.5-turbo")) {
127883
- models.set("gpt-3.5-turbo", pricing);
127884
- }
127885
- if (modelId.includes("claude-3-opus")) {
127886
- models.set("claude-3-opus", pricing);
127887
- }
127888
- if (modelId.includes("claude-3-sonnet")) {
127889
- models.set("claude-3-sonnet", pricing);
127890
- }
127891
- if (modelId.includes("claude-3-haiku")) {
127892
- models.set("claude-3-haiku", pricing);
127893
- }
127894
- if (modelId.includes("gemini-pro")) {
127895
- models.set("gemini-pro", pricing);
127896
- }
127897
- if (modelId.includes("gemini-1.5-pro")) {
127898
- models.set("gemini-1.5-pro", pricing);
127899
- }
127900
- if (modelId.includes("gemini-1.5-flash")) {
127901
- models.set("gemini-1.5-flash", pricing);
127590
+ return latest;
127591
+ }
127592
+ deleteCheckpoint(checkpointId) {
127593
+ const existed = this.mirror.has(checkpointId);
127594
+ this.mirror.delete(checkpointId);
127595
+ this.ensureHydrated();
127596
+ this.chainWrite(checkpointId, async () => {
127597
+ const prisma2 = this.getClient();
127598
+ await prisma2.$executeRaw`
127599
+ DELETE FROM task_checkpoints WHERE id = ${checkpointId}
127600
+ `;
127601
+ });
127602
+ return existed;
127603
+ }
127604
+ purgeExpired() {
127605
+ const now2 = Date.now();
127606
+ let count = 0;
127607
+ const toRemove = [];
127608
+ for (const ckpt of this.mirror.values()) {
127609
+ if (ckpt.expiresAt != null && ckpt.expiresAt < now2) {
127610
+ toRemove.push(ckpt.id);
127611
+ count++;
127902
127612
  }
127903
127613
  }
127904
- clearCache() {
127905
- this.memoryCache = null;
127906
- this.memoryCacheTimestamp = 0;
127907
- logger.debug("Models.dev pricing memory cache cleared");
127614
+ for (const id of toRemove)
127615
+ this.mirror.delete(id);
127616
+ if (count > 0) {
127617
+ this.chainWrite("__purge__", async () => {
127618
+ const prisma2 = this.getClient();
127619
+ await prisma2.$executeRaw`
127620
+ DELETE FROM task_checkpoints
127621
+ WHERE expires_at IS NOT NULL AND expires_at < ${now2}::bigint
127622
+ `;
127623
+ });
127624
+ logger.info("Expired checkpoints purged (PG)", { count });
127908
127625
  }
127909
- async clearAllCaches() {
127910
- this.clearCache();
127911
- const cachePath = this.getLocalCachePath();
127912
- try {
127913
- if (existsSync4(cachePath)) {
127914
- await fs11.unlink(cachePath);
127915
- logger.debug("Local pricing cache file deleted");
127916
- }
127917
- } catch (error51) {
127918
- logger.warn("Failed to delete local pricing cache", {
127919
- error: error51.message
127920
- });
127626
+ return count;
127627
+ }
127628
+ async countExistingMemoryIds(memoryIds) {
127629
+ if (memoryIds.length === 0)
127630
+ return [];
127631
+ this.ensureHydrated();
127632
+ const BATCH_SIZE = 1000;
127633
+ const existing = [];
127634
+ try {
127635
+ const prisma2 = this.getClient();
127636
+ for (let i = 0;i < memoryIds.length; i += BATCH_SIZE) {
127637
+ const batch = memoryIds.slice(i, i + BATCH_SIZE);
127638
+ const rows = await prisma2.$queryRaw`
127639
+ SELECT id FROM memories WHERE id IN (${import_prisma4.Prisma.join(batch)})
127640
+ `;
127641
+ for (const row of rows)
127642
+ existing.push(row.id);
127921
127643
  }
127644
+ return existing;
127645
+ } catch (e) {
127646
+ logger.warn("countExistingMemoryIds failed (best-effort: assuming all exist)", {
127647
+ error: e.message
127648
+ });
127649
+ return memoryIds;
127922
127650
  }
127923
- async hasLocalCache() {
127924
- const cachePath = this.getLocalCachePath();
127925
- return existsSync4(cachePath);
127651
+ }
127652
+ getStats() {
127653
+ this.ensureHydrated();
127654
+ const checkpoints = Array.from(this.mirror.values());
127655
+ const byType = {};
127656
+ let totalSizeBytes = 0;
127657
+ let oldest;
127658
+ for (const c of checkpoints) {
127659
+ byType[c.checkpointType] = (byType[c.checkpointType] ?? 0) + 1;
127660
+ totalSizeBytes += compressState(c.state).byteLength;
127661
+ if (oldest == null || c.createdAt < oldest)
127662
+ oldest = c.createdAt;
127926
127663
  }
127927
- };
127928
- });
127929
-
127930
- // ../../packages/core/dist/services/memory/redundancy-filter.js
127931
- class RedundancyFilter {
127932
- static instance = null;
127933
- static getInstance() {
127934
- return this.instance ??= new RedundancyFilter;
127664
+ return {
127665
+ totalCheckpoints: checkpoints.length,
127666
+ byType,
127667
+ totalSizeBytes,
127668
+ oldestCheckpointAge: oldest != null ? Date.now() - oldest : undefined
127669
+ };
127935
127670
  }
127936
- async findDuplicates(threshold = 0.95, scanLimit = 300) {
127937
- const rows = await getPrismaClient2().$queryRaw`
127938
- SELECT id, content, type, level, importance, array_to_json(tags)::text AS tags, embedding,
127939
- created_at, updated_at, access_count, user_id, session_id, project_id, agent_id
127940
- FROM memories WHERE embedding IS NOT NULL AND deleted_at IS NULL
127941
- ORDER BY created_at DESC LIMIT ${scanLimit}`;
127942
- const groups = new Map;
127943
- for (const row of rows) {
127944
- if (!row.embedding)
127671
+ ensureReady() {
127672
+ return this.ensureHydrated();
127673
+ }
127674
+ close() {}
127675
+ applyFilters(iter, options) {
127676
+ const { taskId, projectId, checkpointType, includeExpired = false, limit = 20, offset = 0 } = options;
127677
+ const now2 = Date.now();
127678
+ const out = [];
127679
+ for (const c of iter) {
127680
+ if (taskId && c.taskId !== taskId)
127945
127681
  continue;
127946
- const bytes = Buffer.from(row.embedding);
127947
- const vector = new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
127948
- let norm = 0;
127949
- for (const value of vector)
127950
- norm += value * value;
127951
- norm = Math.sqrt(norm);
127952
- if (!norm)
127682
+ if (projectId && c.projectId !== projectId)
127953
127683
  continue;
127954
- const group = groups.get(row.type) ?? [];
127955
- group.push({ row, vector, norm });
127956
- groups.set(row.type, group);
127684
+ if (checkpointType && c.checkpointType !== checkpointType)
127685
+ continue;
127686
+ if (!includeExpired && c.expiresAt != null && c.expiresAt <= now2)
127687
+ continue;
127688
+ out.push(c);
127957
127689
  }
127958
- const pairs = [];
127959
- const removed = new Set;
127960
- for (const group of groups.values())
127961
- for (let left = 0;left < group.length; left++)
127962
- for (let right = left + 1;right < group.length; right++) {
127963
- const a = group[left];
127964
- const b = group[right];
127965
- if (removed.has(a.row.id) || removed.has(b.row.id) || a.vector.length !== b.vector.length)
127966
- continue;
127967
- let dot = 0;
127968
- for (let index = 0;index < a.vector.length; index++)
127969
- dot += a.vector[index] * b.vector[index];
127970
- const similarity = dot / (a.norm * b.norm);
127971
- if (similarity < threshold)
127972
- continue;
127973
- const keep = a.row.importance !== b.row.importance ? a.row.importance > b.row.importance ? a : b : a.row.access_count >= b.row.access_count ? a : b;
127974
- const drop = keep === a ? b : a;
127975
- removed.add(drop.row.id);
127976
- pairs.push({ keepId: keep.row.id, removeId: drop.row.id, similarity, reason: keep.row.importance !== drop.row.importance ? "Higher importance" : "Higher access count" });
127977
- }
127978
- return pairs;
127690
+ out.sort((a, b) => b.createdAt - a.createdAt);
127691
+ return out.slice(offset, offset + limit);
127692
+ }
127693
+ checkpointToMetadata(c) {
127694
+ return {
127695
+ id: c.id,
127696
+ taskId: c.taskId,
127697
+ taskDescription: c.taskDescription,
127698
+ agentId: c.agentId,
127699
+ projectId: c.projectId,
127700
+ checkpointType: c.checkpointType,
127701
+ parentCheckpointId: c.parentCheckpointId,
127702
+ createdAt: c.createdAt,
127703
+ expiresAt: c.expiresAt,
127704
+ compressedSizeBytes: compressState(c.state).byteLength,
127705
+ memoryCount: c.memoryIds.length,
127706
+ fileChangeCount: c.fileChanges.length
127707
+ };
127979
127708
  }
127980
- async mergeDuplicates(pairs) {
127981
- if (!pairs.length)
127982
- return { merged: 0, edgesTransferred: 0, accessCountsBoosted: 0 };
127983
- const prisma2 = getPrismaClient2();
127984
- let merged = 0;
127985
- let edgesTransferred = 0;
127986
- let accessCountsBoosted = 0;
127987
- await prisma2.$transaction(async (tx) => {
127988
- for (const pair of pairs) {
127989
- const rows = await tx.$queryRaw`SELECT content, access_count FROM memories WHERE id = ${pair.removeId} FOR UPDATE`;
127990
- if (!rows[0])
127991
- continue;
127992
- const moved = await tx.$executeRaw`
127993
- INSERT INTO memory_edges (from_id, to_id, edge_type, weight, metadata, created_at, updated_at)
127994
- SELECT CASE WHEN from_id = ${pair.removeId} THEN ${pair.keepId} ELSE from_id END,
127995
- CASE WHEN to_id = ${pair.removeId} THEN ${pair.keepId} ELSE to_id END,
127996
- edge_type, weight, metadata, NOW(), NOW()
127997
- FROM memory_edges WHERE (from_id = ${pair.removeId} OR to_id = ${pair.removeId})
127998
- AND from_id <> ${pair.keepId} AND to_id <> ${pair.keepId}
127999
- ON CONFLICT (from_id, to_id, edge_type) DO UPDATE SET weight = GREATEST(memory_edges.weight, EXCLUDED.weight), updated_at = NOW()`;
128000
- await tx.$executeRaw`DELETE FROM memory_edges WHERE from_id = ${pair.removeId} OR to_id = ${pair.removeId}`;
128001
- await tx.$executeRaw`UPDATE memories SET access_count = access_count + ${rows[0].access_count}, updated_at = NOW() WHERE id = ${pair.keepId}`;
128002
- await tx.$executeRaw`DELETE FROM memories WHERE id = ${pair.removeId}`;
128003
- TokenMetrics.getInstance().recordRedundancyFilterSavings(rows[0].content);
128004
- merged++;
128005
- edgesTransferred += Number(moved);
128006
- if (rows[0].access_count > 0)
128007
- accessCountsBoosted++;
128008
- }
127709
+ chainWrite(key, fn) {
127710
+ const prev = this.inflight.get(key) ?? Promise.resolve();
127711
+ const next = prev.then(fn).catch((e) => {
127712
+ logger.warn("PgCheckpointStore write failed (best-effort)", {
127713
+ key,
127714
+ error: e.message
127715
+ });
127716
+ });
127717
+ this.inflight.set(key, next);
127718
+ next.then(() => {
127719
+ if (this.inflight.get(key) === next)
127720
+ this.inflight.delete(key);
128009
127721
  });
128010
- logger.info("RedundancyFilter: merge complete", { merged, edgesTransferred, accessCountsBoosted });
128011
- return { merged, edgesTransferred, accessCountsBoosted };
128012
127722
  }
128013
- async runCleanup(threshold = 0.95) {
128014
- const start = Date.now();
128015
- const pairs = await this.findDuplicates(threshold);
128016
- const result = await this.mergeDuplicates(pairs);
128017
- return { duplicatesFound: pairs.length, merged: result.merged, edgesTransferred: result.edgesTransferred, durationMs: Date.now() - start };
127723
+ async __drain() {
127724
+ const pending = Array.from(this.inflight.values());
127725
+ if (pending.length > 0)
127726
+ await Promise.allSettled(pending);
127727
+ await new Promise((r) => setTimeout(r, 10));
128018
127728
  }
128019
- close() {
128020
- RedundancyFilter.instance = null;
127729
+ async __hydrate() {
127730
+ await this.ensureHydrated();
128021
127731
  }
128022
127732
  }
128023
- var init_redundancy_filter = __esm(() => {
127733
+ var import_prisma4, SUPPORTED_CHECKPOINT_STATE_SCHEMA_VERSION = "1.0.0";
127734
+ var init_checkpoint_store_pg = __esm(() => {
128024
127735
  init_dist();
128025
127736
  init_prisma_client();
128026
- init_token_metrics();
127737
+ init_alias_resolver();
127738
+ init_schema_version();
127739
+ import_prisma4 = __toESM(require_prisma(), 1);
128027
127740
  });
128028
127741
 
128029
- // ../../packages/core/dist/services/memory/memory-clustering.js
128030
- class MemoryClustering {
128031
- static instance = null;
128032
- static getInstance() {
128033
- if (!MemoryClustering.instance) {
128034
- MemoryClustering.instance = new MemoryClustering;
127742
+ // ../../packages/core/dist/services/checkpoint/checkpoint-manager.js
127743
+ var exports_checkpoint_manager = {};
127744
+ __export(exports_checkpoint_manager, {
127745
+ CheckpointManager: () => CheckpointManager
127746
+ });
127747
+ var CheckpointManager;
127748
+ var init_checkpoint_manager = __esm(() => {
127749
+ init_config();
127750
+ init_checkpoint_store_pg();
127751
+ CheckpointManager = class CheckpointManager extends PgCheckpointStore {
127752
+ static instance = null;
127753
+ static getInstance() {
127754
+ requirePostgresDatabaseUrl();
127755
+ return this.instance ??= new CheckpointManager;
128035
127756
  }
128036
- return MemoryClustering.instance;
127757
+ async restoreCheckpoint(checkpointId) {
127758
+ const checkpoint = this.getCheckpoint(checkpointId);
127759
+ if (!checkpoint)
127760
+ return null;
127761
+ const existing = new Set(await this.countExistingMemoryIds(checkpoint.memoryIds));
127762
+ const validMemoryIds = checkpoint.memoryIds.filter((id) => existing.has(id));
127763
+ const missingMemoryIds = checkpoint.memoryIds.filter((id) => !existing.has(id));
127764
+ const fileConflicts = [];
127765
+ const restoreInstructions = [
127766
+ `Restore checkpoint ${checkpoint.id} for task ${checkpoint.taskId}.`,
127767
+ missingMemoryIds.length ? `Missing memories: ${missingMemoryIds.join(", ")}.` : "All referenced memories are available."
127768
+ ].join(`
127769
+ `);
127770
+ return { checkpoint, validMemoryIds, missingMemoryIds, fileConflicts, restoreInstructions };
127771
+ }
127772
+ };
127773
+ });
127774
+
127775
+ // ../../packages/core/dist/services/scheduler/scheduler-defaults.js
127776
+ function envBool2(key, fallback) {
127777
+ const raw2 = process.env[key];
127778
+ if (raw2 === undefined)
127779
+ return fallback;
127780
+ return raw2 === "true" || raw2 === "1";
127781
+ }
127782
+ function envNum2(key, fallback) {
127783
+ const raw2 = process.env[key];
127784
+ if (raw2 === undefined || raw2 === "")
127785
+ return fallback;
127786
+ const n = Number(raw2);
127787
+ return Number.isFinite(n) && n > 0 ? n : fallback;
127788
+ }
127789
+ function applySafeDefaults(job) {
127790
+ if (!envBool2("MASSA_AI_SCHEDULER_SAFE_DEFAULTS", false)) {
127791
+ return job;
128037
127792
  }
128038
- constructor() {}
128039
- async clusterMemories(k, maxIter = 20, maxMemories = 500) {
128040
- const start = Date.now();
128041
- const rows = await getPrismaClient2().$queryRaw`
128042
- SELECT id, content, type, level, importance, array_to_json(tags)::text AS tags,
128043
- embedding, created_at, updated_at, access_count, user_id, session_id, project_id, agent_id
128044
- FROM memories WHERE embedding IS NOT NULL ORDER BY created_at DESC LIMIT ${maxMemories}`;
128045
- const items = [];
128046
- for (const row of rows) {
128047
- if (!row.embedding)
128048
- continue;
128049
- const buf = row.embedding instanceof Buffer ? row.embedding : Buffer.from(row.embedding);
128050
- const vec = Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
128051
- if (vec.every((v) => v === 0))
128052
- continue;
128053
- items.push({ row, vec });
127793
+ if (job.jobKind === "memory-consolidation") {
127794
+ const currentInterval = job.schedule.intervalMs ?? THIRTY_MIN;
127795
+ const safeInterval = Math.max(currentInterval, THIRTY_MIN);
127796
+ return {
127797
+ ...job,
127798
+ defaultEnabled: true,
127799
+ schedule: { type: "interval", intervalMs: safeInterval }
127800
+ };
127801
+ }
127802
+ if (job.jobKind === "decay-sweep") {
127803
+ const currentInterval = job.schedule.intervalMs ?? ONE_HOUR;
127804
+ const safeInterval = Math.max(currentInterval, ONE_HOUR);
127805
+ return {
127806
+ ...job,
127807
+ defaultEnabled: true,
127808
+ schedule: { type: "interval", intervalMs: safeInterval }
127809
+ };
127810
+ }
127811
+ return job;
127812
+ }
127813
+ function registerDefaultJobs(scheduler) {
127814
+ scheduler.registerHandler("memory-consolidation", async () => {
127815
+ const { memoryConsolidationJob: memoryConsolidationJob2 } = await Promise.resolve().then(() => (init_memory_consolidation_job(), exports_memory_consolidation_job));
127816
+ await memoryConsolidationJob2.consolidate();
127817
+ });
127818
+ scheduler.registerHandler("decay-sweep", async () => {
127819
+ const { memoryConsolidationJob: memoryConsolidationJob2 } = await Promise.resolve().then(() => (init_memory_consolidation_job(), exports_memory_consolidation_job));
127820
+ await memoryConsolidationJob2.consolidate();
127821
+ });
127822
+ scheduler.registerHandler("auto-improve", async (job) => {
127823
+ const { autoImproveJob: autoImproveJob2 } = await Promise.resolve().then(() => (init_auto_improve_job(), exports_auto_improve_job));
127824
+ const projectId = job.payload?.projectId ?? "default";
127825
+ await autoImproveJob2.runOnce(projectId);
127826
+ });
127827
+ scheduler.registerHandler("observation-bridge", async (job) => {
127828
+ const { observationConsolidationJob: observationConsolidationJob2 } = await Promise.resolve().then(() => (init_observation_consolidation_job(), exports_observation_consolidation_job));
127829
+ const projectId = job.payload?.projectId ?? "default";
127830
+ await observationConsolidationJob2.runOnce(projectId);
127831
+ });
127832
+ scheduler.registerHandler("checkpoint-purge", async () => {
127833
+ const { CheckpointManager: CheckpointManager2 } = await Promise.resolve().then(() => (init_checkpoint_manager(), exports_checkpoint_manager));
127834
+ const count = CheckpointManager2.getInstance().purgeExpired();
127835
+ logger.info("Scheduled checkpoint purge completed", { count });
127836
+ });
127837
+ for (const rawDef of DEFAULT_SCHEDULED_JOBS) {
127838
+ const def = applySafeDefaults(rawDef);
127839
+ const enabled = envBool2(def.enableEnvVar, def.defaultEnabled);
127840
+ const intervalMs = envNum2(def.intervalEnvVar, def.schedule.intervalMs ?? THIRTY_MIN);
127841
+ const schedule = { type: "interval", intervalMs };
127842
+ scheduler.registerOrResumeJob({
127843
+ id: def.id,
127844
+ name: def.name,
127845
+ jobKind: def.jobKind,
127846
+ schedule,
127847
+ nextRunAt: 0,
127848
+ enabled
127849
+ });
127850
+ }
127851
+ logger.info("Scheduler default jobs registered", {
127852
+ handlers: scheduler.registeredKinds(),
127853
+ jobs: DEFAULT_SCHEDULED_JOBS.length
127854
+ });
127855
+ }
127856
+ var MIN, THIRTY_MIN, ONE_HOUR, DEFAULT_SCHEDULED_JOBS;
127857
+ var init_scheduler_defaults = __esm(() => {
127858
+ init_dist();
127859
+ MIN = 60 * 1000;
127860
+ THIRTY_MIN = 30 * MIN;
127861
+ ONE_HOUR = 60 * MIN;
127862
+ DEFAULT_SCHEDULED_JOBS = [
127863
+ {
127864
+ id: "scheduled-memory-consolidation",
127865
+ name: "Memory Consolidation (clock)",
127866
+ jobKind: "memory-consolidation",
127867
+ schedule: { type: "interval", intervalMs: THIRTY_MIN },
127868
+ defaultEnabled: false,
127869
+ enableEnvVar: "MASSA_AI_SCHEDULER_CONSOLIDATION_ENABLED",
127870
+ intervalEnvVar: "MASSA_AI_SCHEDULER_CONSOLIDATION_INTERVAL_MS"
127871
+ },
127872
+ {
127873
+ id: "scheduled-decay-sweep",
127874
+ name: "Decay Sweep (clock)",
127875
+ jobKind: "decay-sweep",
127876
+ schedule: { type: "interval", intervalMs: ONE_HOUR },
127877
+ defaultEnabled: false,
127878
+ enableEnvVar: "MASSA_AI_SCHEDULER_DECAY_ENABLED",
127879
+ intervalEnvVar: "MASSA_AI_SCHEDULER_DECAY_INTERVAL_MS"
127880
+ },
127881
+ {
127882
+ id: "scheduled-auto-improve",
127883
+ name: "Auto-Improve (clock)",
127884
+ jobKind: "auto-improve",
127885
+ schedule: { type: "interval", intervalMs: THIRTY_MIN },
127886
+ defaultEnabled: false,
127887
+ enableEnvVar: "MASSA_AI_SCHEDULER_AUTO_IMPROVE_ENABLED",
127888
+ intervalEnvVar: "MASSA_AI_SCHEDULER_AUTO_IMPROVE_INTERVAL_MS"
127889
+ },
127890
+ {
127891
+ id: "scheduled-observation-bridge",
127892
+ name: "Observation Bridge (clock)",
127893
+ jobKind: "observation-bridge",
127894
+ schedule: { type: "interval", intervalMs: THIRTY_MIN },
127895
+ defaultEnabled: false,
127896
+ enableEnvVar: "MASSA_AI_SCHEDULER_OBSERVATION_BRIDGE_ENABLED",
127897
+ intervalEnvVar: "MASSA_AI_SCHEDULER_OBSERVATION_BRIDGE_INTERVAL_MS"
127898
+ },
127899
+ {
127900
+ id: "scheduled-checkpoint-purge",
127901
+ name: "Checkpoint Purge (clock)",
127902
+ jobKind: "checkpoint-purge",
127903
+ schedule: { type: "interval", intervalMs: ONE_HOUR },
127904
+ defaultEnabled: false,
127905
+ enableEnvVar: "MASSA_AI_SCHEDULER_CHECKPOINT_PURGE_ENABLED",
127906
+ intervalEnvVar: "MASSA_AI_SCHEDULER_CHECKPOINT_PURGE_INTERVAL_MS"
128054
127907
  }
128055
- if (items.length < 3) {
128056
- return { clusters: [], unclustered: items.length, durationMs: Date.now() - start };
127908
+ ];
127909
+ });
127910
+
127911
+ // ../../packages/core/dist/services/scheduler/index.js
127912
+ var init_scheduler2 = __esm(() => {
127913
+ init_scheduler();
127914
+ init_scheduler_store_pg();
127915
+ init_scheduler_store_factory();
127916
+ init_scheduler_cron();
127917
+ init_scheduler_defaults();
127918
+ });
127919
+
127920
+ // ../../packages/core/dist/services/pricing/models-dev-client.js
127921
+ import fs11 from "fs/promises";
127922
+ import { existsSync as existsSync4 } from "fs";
127923
+ import path16 from "path";
127924
+ function getModelsDevClient() {
127925
+ if (!clientInstance) {
127926
+ clientInstance = new ModelsDevClient;
127927
+ }
127928
+ return clientInstance;
127929
+ }
127930
+ var ModelsDevClient, clientInstance = null;
127931
+ var init_models_dev_client = __esm(() => {
127932
+ init_dist();
127933
+ init_dist();
127934
+ ModelsDevClient = class ModelsDevClient {
127935
+ static API_URL = "https://models.dev/api.json";
127936
+ static MEMORY_CACHE_TTL = 3600 * 1000;
127937
+ static LOCAL_CACHE_TTL = 24 * 3600 * 1000;
127938
+ static LOCAL_CACHE_FILE = "pricing-cache.json";
127939
+ memoryCache = null;
127940
+ memoryCacheTimestamp = 0;
127941
+ getLocalCachePath() {
127942
+ const dataDir = config.get("dataDir");
127943
+ return path16.join(dataDir, ModelsDevClient.LOCAL_CACHE_FILE);
128057
127944
  }
128058
- const effectiveK = k ?? Math.max(2, Math.min(20, Math.round(Math.sqrt(items.length / 2))));
128059
- const dim = items[0].vec.length;
128060
- let centroids = this.kMeansPlusPlusInit(items.map((i) => i.vec), effectiveK);
128061
- let assignments = new Int32Array(items.length);
128062
- for (let iter = 0;iter < maxIter; iter++) {
128063
- const newAssignments = new Int32Array(items.length);
128064
- for (let i = 0;i < items.length; i++) {
128065
- let bestDist = Infinity;
128066
- let bestCluster = 0;
128067
- for (let c = 0;c < centroids.length; c++) {
128068
- const dist = this.euclideanDistanceSq(items[i].vec, centroids[c]);
128069
- if (dist < bestDist) {
128070
- bestDist = dist;
128071
- bestCluster = c;
128072
- }
128073
- }
128074
- newAssignments[i] = bestCluster;
128075
- }
128076
- let changed = false;
128077
- for (let i = 0;i < items.length; i++) {
128078
- if (newAssignments[i] !== assignments[i]) {
128079
- changed = true;
128080
- break;
127945
+ async loadLocalCache() {
127946
+ const cachePath = this.getLocalCachePath();
127947
+ try {
127948
+ if (!existsSync4(cachePath)) {
127949
+ return null;
128081
127950
  }
128082
- }
128083
- assignments = newAssignments;
128084
- if (!changed)
128085
- break;
128086
- const newCentroids = [];
128087
- const counts = [];
128088
- for (let c = 0;c < centroids.length; c++) {
128089
- newCentroids.push(Array.from({ length: dim }, () => 0));
128090
- counts.push(0);
128091
- }
128092
- for (let i = 0;i < items.length; i++) {
128093
- const c = assignments[i];
128094
- counts[c]++;
128095
- for (let d = 0;d < dim; d++) {
128096
- newCentroids[c][d] += items[i].vec[d];
127951
+ const content = await fs11.readFile(cachePath, "utf-8");
127952
+ const data = JSON.parse(content);
127953
+ const age = Date.now() - data.timestamp;
127954
+ if (age > ModelsDevClient.LOCAL_CACHE_TTL) {
127955
+ logger.debug("Local pricing cache expired", {
127956
+ ageHours: Math.round(age / 3600000)
127957
+ });
128097
127958
  }
128098
- }
128099
- for (let c = 0;c < centroids.length; c++) {
128100
- if (counts[c] === 0)
128101
- continue;
128102
- for (let d = 0;d < dim; d++) {
128103
- newCentroids[c][d] /= counts[c];
127959
+ const models = new Map;
127960
+ for (const [key, value] of Object.entries(data.models)) {
127961
+ models.set(key, value);
128104
127962
  }
127963
+ logger.debug("Loaded pricing from local cache", {
127964
+ models: models.size,
127965
+ ageHours: Math.round(age / 3600000)
127966
+ });
127967
+ return models;
127968
+ } catch (error51) {
127969
+ logger.debug("Failed to load local pricing cache", {
127970
+ error: error51.message
127971
+ });
127972
+ return null;
128105
127973
  }
128106
- centroids = newCentroids;
128107
- }
128108
- const clusterMap = new Map;
128109
- for (let i = 0;i < items.length; i++) {
128110
- const c = assignments[i];
128111
- if (!clusterMap.has(c))
128112
- clusterMap.set(c, []);
128113
- clusterMap.get(c).push(items[i].row);
128114
- }
128115
- const clusters = [];
128116
- let unclustered = 0;
128117
- for (const [cIdx, members] of clusterMap) {
128118
- if (members.length < 2) {
128119
- unclustered += members.length;
128120
- continue;
128121
- }
128122
- const label = this.generateLabel(members);
128123
- const avgImportance = members.reduce((sum, m) => sum + m.importance, 0) / members.length;
128124
- const totalAccess = members.reduce((sum, m) => sum + m.access_count, 0);
128125
- const dominantType = this.getDominantType(members);
128126
- clusters.push({
128127
- id: `cluster_${Date.now()}_${cIdx}`,
128128
- centroid: centroids[cIdx],
128129
- memberIds: members.map((m) => m.id),
128130
- label,
128131
- importance: Math.round(avgImportance * 100) / 100,
128132
- totalAccess,
128133
- dominantType
128134
- });
128135
127974
  }
128136
- clusters.sort((a, b) => b.importance - a.importance);
128137
- logger.info("MemoryClustering: complete", {
128138
- inputMemories: items.length,
128139
- clusters: clusters.length,
128140
- unclustered,
128141
- k: effectiveK
128142
- });
128143
- return { clusters, unclustered, durationMs: Date.now() - start };
128144
- }
128145
- async findCluster(memoryId, cached2) {
128146
- const result = cached2 ?? await this.clusterMemories();
128147
- for (const cluster of result.clusters) {
128148
- if (cluster.memberIds.includes(memoryId)) {
128149
- return cluster;
127975
+ async saveLocalCache(models) {
127976
+ const cachePath = this.getLocalCachePath();
127977
+ try {
127978
+ const dir = path16.dirname(cachePath);
127979
+ await fs11.mkdir(dir, { recursive: true });
127980
+ const data = {
127981
+ timestamp: Date.now(),
127982
+ version: "1.0.0",
127983
+ models: Object.fromEntries(models)
127984
+ };
127985
+ await fs11.writeFile(cachePath, JSON.stringify(data), "utf-8");
127986
+ logger.debug("Saved pricing to local cache", {
127987
+ models: models.size,
127988
+ path: cachePath
127989
+ });
127990
+ } catch (error51) {
127991
+ logger.warn("Failed to save local pricing cache", {
127992
+ error: error51.message
127993
+ });
128150
127994
  }
128151
127995
  }
128152
- return null;
128153
- }
128154
- async summarizeCluster(cluster) {
128155
- const members = await getPrismaClient2().$queryRaw`SELECT content, type, importance FROM memories WHERE id = ANY(${cluster.memberIds}::text[]) ORDER BY importance DESC LIMIT 5`;
128156
- if (members.length === 0)
128157
- return cluster.label;
128158
- const lead = members[0].content.split(/[.!?\n]/)[0].trim();
128159
- const typeCount = members.length;
128160
- const types6 = [...new Set(members.map((m) => m.type))].join(", ");
128161
- return `[${cluster.label}] ${lead} (${typeCount} memories, types: ${types6})`;
128162
- }
128163
- kMeansPlusPlusInit(vectors, k) {
128164
- const centroids = [];
128165
- const firstIdx = Math.floor(Math.random() * vectors.length);
128166
- centroids.push([...vectors[firstIdx]]);
128167
- for (let c = 1;c < k; c++) {
128168
- const distances = [];
128169
- let totalDist = 0;
128170
- for (const vec of vectors) {
128171
- let minDist = Infinity;
128172
- for (const centroid of centroids) {
128173
- const d = this.euclideanDistanceSq(vec, centroid);
128174
- if (d < minDist)
128175
- minDist = d;
128176
- }
128177
- distances.push(minDist);
128178
- totalDist += minDist;
128179
- }
128180
- if (totalDist === 0) {
128181
- centroids.push([...vectors[Math.floor(Math.random() * vectors.length)]]);
128182
- continue;
128183
- }
128184
- let r = Math.random() * totalDist;
128185
- for (let i = 0;i < distances.length; i++) {
128186
- r -= distances[i];
128187
- if (r <= 0) {
128188
- centroids.push([...vectors[i]]);
128189
- break;
127996
+ getBundledDefaults() {
127997
+ const models = new Map;
127998
+ const defaults3 = [
127999
+ {
128000
+ id: "ollama/nomic-embed-text",
128001
+ name: "Nomic Embed Text",
128002
+ provider: "Ollama",
128003
+ family: "nomic",
128004
+ inputCostPerMillion: 0,
128005
+ outputCostPerMillion: 0,
128006
+ contextWindow: 8192
128007
+ },
128008
+ {
128009
+ id: "ollama/all-minilm",
128010
+ name: "All MiniLM",
128011
+ provider: "Ollama",
128012
+ family: "minilm",
128013
+ inputCostPerMillion: 0,
128014
+ outputCostPerMillion: 0,
128015
+ contextWindow: 512
128016
+ },
128017
+ {
128018
+ id: "ollama/mxbai-embed-large",
128019
+ name: "MxBai Embed Large",
128020
+ provider: "Ollama",
128021
+ family: "mxbai",
128022
+ inputCostPerMillion: 0,
128023
+ outputCostPerMillion: 0,
128024
+ contextWindow: 512
128025
+ },
128026
+ {
128027
+ id: "mistral-embed",
128028
+ name: "Mistral Embed",
128029
+ provider: "Mistral",
128030
+ family: "mistral",
128031
+ inputCostPerMillion: 0.1,
128032
+ outputCostPerMillion: 0,
128033
+ contextWindow: 8192
128034
+ },
128035
+ {
128036
+ id: "codestral-embed",
128037
+ name: "Codestral Embed",
128038
+ provider: "Mistral",
128039
+ family: "codestral",
128040
+ inputCostPerMillion: 0.1,
128041
+ outputCostPerMillion: 0,
128042
+ contextWindow: 32768
128043
+ },
128044
+ {
128045
+ id: "mistral-small-latest",
128046
+ name: "Mistral Small",
128047
+ provider: "Mistral",
128048
+ family: "mistral",
128049
+ inputCostPerMillion: 0.2,
128050
+ outputCostPerMillion: 0.6,
128051
+ contextWindow: 32768,
128052
+ maxOutputTokens: 8192
128053
+ },
128054
+ {
128055
+ id: "text-embedding-3-small",
128056
+ name: "Text Embedding 3 Small",
128057
+ provider: "OpenAI",
128058
+ family: "embedding",
128059
+ inputCostPerMillion: 0.02,
128060
+ outputCostPerMillion: 0,
128061
+ contextWindow: 8191
128062
+ },
128063
+ {
128064
+ id: "gpt-4o-mini",
128065
+ name: "GPT-4o Mini",
128066
+ provider: "OpenAI",
128067
+ family: "gpt-4",
128068
+ inputCostPerMillion: 0.15,
128069
+ outputCostPerMillion: 0.6,
128070
+ contextWindow: 128000,
128071
+ maxOutputTokens: 16384
128072
+ },
128073
+ {
128074
+ id: "claude-3-haiku-20240307",
128075
+ name: "Claude 3 Haiku",
128076
+ provider: "Anthropic",
128077
+ family: "claude-3",
128078
+ inputCostPerMillion: 0.25,
128079
+ outputCostPerMillion: 1.25,
128080
+ contextWindow: 200000,
128081
+ maxOutputTokens: 4096
128082
+ },
128083
+ {
128084
+ id: "claude-3-5-sonnet-20241022",
128085
+ name: "Claude 3.5 Sonnet",
128086
+ provider: "Anthropic",
128087
+ family: "claude-3.5",
128088
+ inputCostPerMillion: 3,
128089
+ outputCostPerMillion: 15,
128090
+ contextWindow: 200000,
128091
+ maxOutputTokens: 8192
128092
+ },
128093
+ {
128094
+ id: "gemini-1.5-flash",
128095
+ name: "Gemini 1.5 Flash",
128096
+ provider: "Google",
128097
+ family: "gemini",
128098
+ inputCostPerMillion: 0.075,
128099
+ outputCostPerMillion: 0.3,
128100
+ contextWindow: 1e6,
128101
+ maxOutputTokens: 8192
128102
+ },
128103
+ {
128104
+ id: "embed-english-v3.0",
128105
+ name: "Embed English v3",
128106
+ provider: "Cohere",
128107
+ family: "embed",
128108
+ inputCostPerMillion: 0.1,
128109
+ outputCostPerMillion: 0,
128110
+ contextWindow: 512
128190
128111
  }
128112
+ ];
128113
+ for (const pricing of defaults3) {
128114
+ models.set(pricing.id, pricing);
128191
128115
  }
128192
- if (centroids.length <= c) {
128193
- centroids.push([...vectors[vectors.length - 1]]);
128194
- }
128116
+ logger.info("Using bundled pricing defaults (offline mode)", {
128117
+ models: models.size
128118
+ });
128119
+ return models;
128195
128120
  }
128196
- return centroids;
128197
- }
128198
- generateLabel(members) {
128199
- const stopWords = new Set([
128200
- "the",
128201
- "a",
128202
- "an",
128203
- "is",
128204
- "are",
128205
- "was",
128206
- "were",
128207
- "be",
128208
- "been",
128209
- "being",
128210
- "have",
128211
- "has",
128212
- "had",
128213
- "do",
128214
- "does",
128215
- "did",
128216
- "will",
128217
- "would",
128218
- "could",
128219
- "should",
128220
- "may",
128221
- "might",
128222
- "shall",
128223
- "can",
128224
- "need",
128225
- "dare",
128226
- "ought",
128227
- "used",
128228
- "to",
128229
- "of",
128230
- "in",
128231
- "for",
128232
- "on",
128233
- "with",
128234
- "at",
128235
- "by",
128236
- "from",
128237
- "as",
128238
- "into",
128239
- "through",
128240
- "during",
128241
- "before",
128242
- "after",
128243
- "above",
128244
- "below",
128245
- "between",
128246
- "out",
128247
- "off",
128248
- "over",
128249
- "under",
128250
- "again",
128251
- "further",
128252
- "then",
128253
- "once",
128254
- "here",
128255
- "there",
128256
- "when",
128257
- "where",
128258
- "why",
128259
- "how",
128260
- "all",
128261
- "both",
128262
- "each",
128263
- "few",
128264
- "more",
128265
- "most",
128266
- "other",
128267
- "some",
128268
- "such",
128269
- "no",
128270
- "nor",
128271
- "not",
128272
- "only",
128273
- "own",
128274
- "same",
128275
- "so",
128276
- "than",
128277
- "too",
128278
- "very",
128279
- "just",
128280
- "don",
128281
- "should",
128282
- "now",
128283
- "and",
128284
- "but",
128285
- "or",
128286
- "if",
128287
- "while",
128288
- "that",
128289
- "this",
128290
- "it",
128291
- "its",
128292
- "what",
128293
- "which",
128294
- "who",
128295
- "whom",
128296
- "these",
128297
- "those",
128298
- "i",
128299
- "me",
128300
- "my",
128301
- "myself",
128302
- "we",
128303
- "our",
128304
- "ours",
128305
- "you",
128306
- "your",
128307
- "he",
128308
- "him",
128309
- "his",
128310
- "she",
128311
- "her",
128312
- "they",
128313
- "them",
128314
- "their",
128315
- "de",
128316
- "da",
128317
- "do",
128318
- "das",
128319
- "dos",
128320
- "em",
128321
- "no",
128322
- "na",
128323
- "nos",
128324
- "nas",
128325
- "um",
128326
- "uma",
128327
- "uns",
128328
- "umas",
128329
- "para",
128330
- "com",
128331
- "por",
128332
- "que",
128333
- "se",
128334
- "como",
128335
- "mas",
128336
- "ou",
128337
- "quando",
128338
- "mais",
128339
- "tamb\xE9m",
128340
- "j\xE1",
128341
- "ainda",
128342
- "sobre",
128343
- "entre",
128344
- "at\xE9",
128345
- "sem",
128346
- "sob",
128347
- "esse",
128348
- "essa",
128349
- "este",
128350
- "esta",
128351
- "aquele",
128352
- "aquela",
128353
- "ele",
128354
- "ela",
128355
- "eles",
128356
- "elas",
128357
- "n\xF3s",
128358
- "eu",
128359
- "tu",
128360
- "voc\xEA",
128361
- "voc\xEAs",
128362
- "meu",
128363
- "minha",
128364
- "seu",
128365
- "sua"
128366
- ]);
128367
- const wordFreq = new Map;
128368
- for (const member of members) {
128369
- const words = member.content.toLowerCase().replace(/[^a-z\u00E1\u00E0\u00E2\u00E3\u00E9\u00E8\u00EA\u00ED\u00EF\u00F3\u00F4\u00F5\u00FA\u00FC\u00E7\s-]/g, " ").split(/\s+/).filter((w) => w.length > 2 && !stopWords.has(w));
128121
+ async fetchAllModels() {
128122
+ if (this.memoryCache && Date.now() - this.memoryCacheTimestamp < ModelsDevClient.MEMORY_CACHE_TTL) {
128123
+ logger.debug("Using in-memory pricing cache");
128124
+ return this.memoryCache;
128125
+ }
128126
+ const localCache = await this.loadLocalCache();
128127
+ let remoteModels = null;
128128
+ try {
128129
+ remoteModels = await this.fetchRemote();
128130
+ await this.saveLocalCache(remoteModels);
128131
+ } catch (error51) {
128132
+ logger.debug("Remote pricing fetch failed (offline?)", {
128133
+ error: error51.message
128134
+ });
128135
+ }
128136
+ let result;
128137
+ if (remoteModels) {
128138
+ result = remoteModels;
128139
+ logger.debug("Using remote pricing data");
128140
+ } else if (localCache) {
128141
+ result = localCache;
128142
+ logger.info("Using local pricing cache (offline mode)");
128143
+ } else {
128144
+ result = this.getBundledDefaults();
128145
+ }
128146
+ this.memoryCache = result;
128147
+ this.memoryCacheTimestamp = Date.now();
128148
+ return result;
128149
+ }
128150
+ async fetchRemote() {
128151
+ const controller = new AbortController;
128152
+ const timeout = setTimeout(() => controller.abort(), 1e4);
128153
+ try {
128154
+ logger.info("Fetching pricing data from models.dev API");
128155
+ const response = await fetch(ModelsDevClient.API_URL, {
128156
+ signal: controller.signal
128157
+ });
128158
+ if (!response.ok) {
128159
+ throw new Error(`HTTP ${response.status}: ${response.statusText}`);
128160
+ }
128161
+ const data = await response.json();
128162
+ const models = new Map;
128163
+ let totalModels = 0;
128164
+ let modelsWithPricing = 0;
128165
+ for (const [providerId, provider] of Object.entries(data)) {
128166
+ for (const [modelId, model] of Object.entries(provider.models || {})) {
128167
+ totalModels++;
128168
+ const cost = model.cost || { input: 0, output: 0 };
128169
+ if (cost.input === 0 && cost.output === 0) {
128170
+ continue;
128171
+ }
128172
+ modelsWithPricing++;
128173
+ const normalizedId = this.normalizeModelId(modelId);
128174
+ const pricing = {
128175
+ id: normalizedId,
128176
+ name: model.name || modelId,
128177
+ provider: provider.name || providerId,
128178
+ family: model.family,
128179
+ inputCostPerMillion: cost.input,
128180
+ outputCostPerMillion: cost.output,
128181
+ cacheReadCostPerMillion: cost.cache_read,
128182
+ contextWindow: model.limit?.context,
128183
+ maxOutputTokens: model.limit?.output,
128184
+ lastUpdated: model.last_updated
128185
+ };
128186
+ models.set(modelId, pricing);
128187
+ if (normalizedId !== modelId) {
128188
+ models.set(normalizedId, pricing);
128189
+ }
128190
+ this.addCommonAliases(models, modelId, pricing);
128191
+ }
128192
+ }
128193
+ logger.info(`Loaded ${modelsWithPricing} models with pricing (out of ${totalModels} total)`);
128194
+ return models;
128195
+ } finally {
128196
+ clearTimeout(timeout);
128197
+ }
128198
+ }
128199
+ async getModelPricing(modelId) {
128200
+ const models = await this.fetchAllModels();
128201
+ let pricing = models.get(modelId);
128202
+ if (pricing)
128203
+ return pricing;
128204
+ const normalized = this.normalizeModelId(modelId);
128205
+ pricing = models.get(normalized);
128206
+ if (pricing)
128207
+ return pricing;
128208
+ const lowerModelId = modelId.toLowerCase();
128209
+ for (const [key, value] of models.entries()) {
128210
+ if (key.toLowerCase() === lowerModelId) {
128211
+ return value;
128212
+ }
128213
+ }
128214
+ logger.warn(`Model pricing not found: ${modelId}`);
128215
+ return null;
128216
+ }
128217
+ async searchModels(query) {
128218
+ const models = await this.fetchAllModels();
128219
+ const lowerQuery = query.toLowerCase();
128220
+ const results = [];
128370
128221
  const seen = new Set;
128371
- for (const word of words) {
128372
- if (seen.has(word))
128222
+ for (const pricing of models.values()) {
128223
+ const key = `${pricing.provider}:${pricing.name}`;
128224
+ if (seen.has(key))
128373
128225
  continue;
128374
- seen.add(word);
128375
- wordFreq.set(word, (wordFreq.get(word) ?? 0) + 1);
128226
+ seen.add(key);
128227
+ if (pricing.id.toLowerCase().includes(lowerQuery) || pricing.name.toLowerCase().includes(lowerQuery) || pricing.provider.toLowerCase().includes(lowerQuery) || pricing.family && pricing.family.toLowerCase().includes(lowerQuery)) {
128228
+ results.push(pricing);
128229
+ }
128376
128230
  }
128231
+ results.sort((a, b) => b.inputCostPerMillion - a.inputCostPerMillion);
128232
+ return results;
128377
128233
  }
128378
- const sorted = [...wordFreq.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(([word]) => word);
128379
- return sorted.length > 0 ? sorted.join(", ") : "misc";
128234
+ async getTopExpensiveModels(limit = 10) {
128235
+ const models = await this.fetchAllModels();
128236
+ const unique = new Map;
128237
+ for (const pricing of models.values()) {
128238
+ const key = `${pricing.provider}:${pricing.name}`;
128239
+ if (!unique.has(key)) {
128240
+ unique.set(key, pricing);
128241
+ }
128242
+ }
128243
+ return Array.from(unique.values()).sort((a, b) => b.inputCostPerMillion - a.inputCostPerMillion).slice(0, limit);
128244
+ }
128245
+ async getStatistics() {
128246
+ const models = await this.fetchAllModels();
128247
+ const unique = new Map;
128248
+ const providers = new Set;
128249
+ let totalInputCost = 0;
128250
+ let totalOutputCost = 0;
128251
+ for (const pricing of models.values()) {
128252
+ const key = `${pricing.provider}:${pricing.name}`;
128253
+ if (!unique.has(key)) {
128254
+ unique.set(key, pricing);
128255
+ providers.add(pricing.provider);
128256
+ totalInputCost += pricing.inputCostPerMillion;
128257
+ totalOutputCost += pricing.outputCostPerMillion;
128258
+ }
128259
+ }
128260
+ const modelList = Array.from(unique.values());
128261
+ modelList.sort((a, b) => a.inputCostPerMillion - b.inputCostPerMillion);
128262
+ return {
128263
+ totalModels: modelList.length,
128264
+ totalProviders: providers.size,
128265
+ avgInputCost: modelList.length > 0 ? totalInputCost / modelList.length : 0,
128266
+ avgOutputCost: modelList.length > 0 ? totalOutputCost / modelList.length : 0,
128267
+ cheapestModel: modelList[0] || null,
128268
+ mostExpensiveModel: modelList[modelList.length - 1] || null
128269
+ };
128270
+ }
128271
+ normalizeModelId(modelId) {
128272
+ let normalized = modelId.replace(/^(openai|anthropic|google|meta|mistral|cohere)\//i, "").replace(/^(claude-|gpt-|gemini-|llama-)/i, (match2) => match2.toLowerCase());
128273
+ return normalized;
128274
+ }
128275
+ addCommonAliases(models, modelId, pricing) {
128276
+ if (modelId.includes("gpt-4") && !modelId.includes("turbo")) {
128277
+ models.set("gpt-4", pricing);
128278
+ }
128279
+ if (modelId.includes("gpt-4-turbo")) {
128280
+ models.set("gpt-4-turbo", pricing);
128281
+ }
128282
+ if (modelId.includes("gpt-3.5-turbo")) {
128283
+ models.set("gpt-3.5-turbo", pricing);
128284
+ }
128285
+ if (modelId.includes("claude-3-opus")) {
128286
+ models.set("claude-3-opus", pricing);
128287
+ }
128288
+ if (modelId.includes("claude-3-sonnet")) {
128289
+ models.set("claude-3-sonnet", pricing);
128290
+ }
128291
+ if (modelId.includes("claude-3-haiku")) {
128292
+ models.set("claude-3-haiku", pricing);
128293
+ }
128294
+ if (modelId.includes("gemini-pro")) {
128295
+ models.set("gemini-pro", pricing);
128296
+ }
128297
+ if (modelId.includes("gemini-1.5-pro")) {
128298
+ models.set("gemini-1.5-pro", pricing);
128299
+ }
128300
+ if (modelId.includes("gemini-1.5-flash")) {
128301
+ models.set("gemini-1.5-flash", pricing);
128302
+ }
128303
+ }
128304
+ clearCache() {
128305
+ this.memoryCache = null;
128306
+ this.memoryCacheTimestamp = 0;
128307
+ logger.debug("Models.dev pricing memory cache cleared");
128308
+ }
128309
+ async clearAllCaches() {
128310
+ this.clearCache();
128311
+ const cachePath = this.getLocalCachePath();
128312
+ try {
128313
+ if (existsSync4(cachePath)) {
128314
+ await fs11.unlink(cachePath);
128315
+ logger.debug("Local pricing cache file deleted");
128316
+ }
128317
+ } catch (error51) {
128318
+ logger.warn("Failed to delete local pricing cache", {
128319
+ error: error51.message
128320
+ });
128321
+ }
128322
+ }
128323
+ async hasLocalCache() {
128324
+ const cachePath = this.getLocalCachePath();
128325
+ return existsSync4(cachePath);
128326
+ }
128327
+ };
128328
+ });
128329
+
128330
+ // ../../packages/core/dist/services/memory/redundancy-filter.js
128331
+ class RedundancyFilter {
128332
+ static instance = null;
128333
+ static getInstance() {
128334
+ return this.instance ??= new RedundancyFilter;
128380
128335
  }
128381
- getDominantType(members) {
128382
- const typeCounts = new Map;
128383
- for (const m of members) {
128384
- typeCounts.set(m.type, (typeCounts.get(m.type) ?? 0) + 1);
128336
+ async findDuplicates(threshold = 0.95, scanLimit = 300) {
128337
+ const rows = await getPrismaClient2().$queryRaw`
128338
+ SELECT id, content, type, level, importance, array_to_json(tags)::text AS tags, embedding,
128339
+ created_at, updated_at, access_count, user_id, session_id, project_id, agent_id
128340
+ FROM memories WHERE embedding IS NOT NULL AND deleted_at IS NULL
128341
+ ORDER BY created_at DESC LIMIT ${scanLimit}`;
128342
+ const groups = new Map;
128343
+ for (const row of rows) {
128344
+ if (!row.embedding)
128345
+ continue;
128346
+ const bytes = Buffer.from(row.embedding);
128347
+ const vector = new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
128348
+ let norm = 0;
128349
+ for (const value of vector)
128350
+ norm += value * value;
128351
+ norm = Math.sqrt(norm);
128352
+ if (!norm)
128353
+ continue;
128354
+ const group = groups.get(row.type) ?? [];
128355
+ group.push({ row, vector, norm });
128356
+ groups.set(row.type, group);
128385
128357
  }
128386
- let dominant = "unknown";
128387
- let maxCount = 0;
128388
- for (const [type, count] of typeCounts) {
128389
- if (count > maxCount) {
128390
- maxCount = count;
128391
- dominant = type;
128358
+ const pairs = [];
128359
+ const removed = new Set;
128360
+ for (const group of groups.values())
128361
+ for (let left = 0;left < group.length; left++)
128362
+ for (let right = left + 1;right < group.length; right++) {
128363
+ const a = group[left];
128364
+ const b = group[right];
128365
+ if (removed.has(a.row.id) || removed.has(b.row.id) || a.vector.length !== b.vector.length)
128366
+ continue;
128367
+ let dot = 0;
128368
+ for (let index = 0;index < a.vector.length; index++)
128369
+ dot += a.vector[index] * b.vector[index];
128370
+ const similarity = dot / (a.norm * b.norm);
128371
+ if (similarity < threshold)
128372
+ continue;
128373
+ const keep = a.row.importance !== b.row.importance ? a.row.importance > b.row.importance ? a : b : a.row.access_count >= b.row.access_count ? a : b;
128374
+ const drop = keep === a ? b : a;
128375
+ removed.add(drop.row.id);
128376
+ pairs.push({ keepId: keep.row.id, removeId: drop.row.id, similarity, reason: keep.row.importance !== drop.row.importance ? "Higher importance" : "Higher access count" });
128377
+ }
128378
+ return pairs;
128379
+ }
128380
+ async mergeDuplicates(pairs) {
128381
+ if (!pairs.length)
128382
+ return { merged: 0, edgesTransferred: 0, accessCountsBoosted: 0 };
128383
+ const prisma2 = getPrismaClient2();
128384
+ let merged = 0;
128385
+ let edgesTransferred = 0;
128386
+ let accessCountsBoosted = 0;
128387
+ await prisma2.$transaction(async (tx) => {
128388
+ for (const pair of pairs) {
128389
+ const rows = await tx.$queryRaw`SELECT content, access_count FROM memories WHERE id = ${pair.removeId} FOR UPDATE`;
128390
+ if (!rows[0])
128391
+ continue;
128392
+ const moved = await tx.$executeRaw`
128393
+ INSERT INTO memory_edges (from_id, to_id, edge_type, weight, metadata, created_at, updated_at)
128394
+ SELECT CASE WHEN from_id = ${pair.removeId} THEN ${pair.keepId} ELSE from_id END,
128395
+ CASE WHEN to_id = ${pair.removeId} THEN ${pair.keepId} ELSE to_id END,
128396
+ edge_type, weight, metadata, NOW(), NOW()
128397
+ FROM memory_edges WHERE (from_id = ${pair.removeId} OR to_id = ${pair.removeId})
128398
+ AND from_id <> ${pair.keepId} AND to_id <> ${pair.keepId}
128399
+ ON CONFLICT (from_id, to_id, edge_type) DO UPDATE SET weight = GREATEST(memory_edges.weight, EXCLUDED.weight), updated_at = NOW()`;
128400
+ await tx.$executeRaw`DELETE FROM memory_edges WHERE from_id = ${pair.removeId} OR to_id = ${pair.removeId}`;
128401
+ await tx.$executeRaw`UPDATE memories SET access_count = access_count + ${rows[0].access_count}, updated_at = NOW() WHERE id = ${pair.keepId}`;
128402
+ await tx.$executeRaw`DELETE FROM memories WHERE id = ${pair.removeId}`;
128403
+ TokenMetrics.getInstance().recordRedundancyFilterSavings(rows[0].content);
128404
+ merged++;
128405
+ edgesTransferred += Number(moved);
128406
+ if (rows[0].access_count > 0)
128407
+ accessCountsBoosted++;
128392
128408
  }
128393
- }
128394
- return dominant;
128409
+ });
128410
+ logger.info("RedundancyFilter: merge complete", { merged, edgesTransferred, accessCountsBoosted });
128411
+ return { merged, edgesTransferred, accessCountsBoosted };
128395
128412
  }
128396
- euclideanDistanceSq(a, b) {
128397
- let sum = 0;
128398
- for (let i = 0;i < a.length; i++) {
128399
- const diff = a[i] - b[i];
128400
- sum += diff * diff;
128401
- }
128402
- return sum;
128413
+ async runCleanup(threshold = 0.95) {
128414
+ const start = Date.now();
128415
+ const pairs = await this.findDuplicates(threshold);
128416
+ const result = await this.mergeDuplicates(pairs);
128417
+ return { duplicatesFound: pairs.length, merged: result.merged, edgesTransferred: result.edgesTransferred, durationMs: Date.now() - start };
128403
128418
  }
128404
128419
  close() {
128405
- MemoryClustering.instance = null;
128420
+ RedundancyFilter.instance = null;
128406
128421
  }
128407
128422
  }
128408
- var init_memory_clustering = __esm(() => {
128423
+ var init_redundancy_filter = __esm(() => {
128409
128424
  init_dist();
128410
128425
  init_prisma_client();
128426
+ init_token_metrics();
128411
128427
  });
128412
128428
 
128413
- // ../../packages/core/dist/services/checkpoint/checkpoint-store-pg.js
128414
- function toNum4(v) {
128415
- if (v == null)
128416
- return null;
128417
- return typeof v === "bigint" ? Number(v) : v;
128418
- }
128419
- function compressState(state) {
128420
- const json3 = JSON.stringify(state);
128421
- return Buffer.from(Bun.deflateSync(Buffer.from(json3, "utf-8")));
128422
- }
128423
- function decompressState(data) {
128424
- const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
128425
- const inflated = Bun.inflateSync(new Uint8Array(buf));
128426
- return JSON.parse(Buffer.from(inflated).toString("utf-8"));
128427
- }
128428
-
128429
- class PgCheckpointStore {
128430
- prisma;
128431
- mirror = new Map;
128432
- hydrated = false;
128433
- hydrating = null;
128434
- hydrateFailedAt = 0;
128435
- static HYDRATE_RETRY_MS = 30000;
128436
- inflight = new Map;
128437
- getClient() {
128438
- if (!this.prisma)
128439
- this.prisma = getPrismaClient2();
128440
- return this.prisma;
128441
- }
128442
- ensureHydrated() {
128443
- if (this.hydrated)
128444
- return Promise.resolve();
128445
- if (this.hydrating)
128446
- return this.hydrating;
128447
- if (this.hydrateFailedAt > 0 && Date.now() - this.hydrateFailedAt < PgCheckpointStore.HYDRATE_RETRY_MS) {
128448
- return Promise.resolve();
128429
+ // ../../packages/core/dist/services/memory/memory-clustering.js
128430
+ class MemoryClustering {
128431
+ static instance = null;
128432
+ static getInstance() {
128433
+ if (!MemoryClustering.instance) {
128434
+ MemoryClustering.instance = new MemoryClustering;
128449
128435
  }
128450
- this.hydrating = (async () => {
128451
- try {
128452
- const prisma2 = this.getClient();
128453
- const rows = await prisma2.$queryRaw`
128454
- SELECT * FROM task_checkpoints
128455
- `;
128456
- const next = new Map;
128457
- const dbIds = new Set;
128458
- for (const row of rows) {
128459
- dbIds.add(row.id);
128460
- next.set(row.id, this.rowToCheckpoint(row));
128461
- }
128462
- for (const [id, existing] of this.mirror) {
128463
- if (!dbIds.has(id))
128464
- next.set(id, existing);
128465
- }
128466
- this.mirror = next;
128467
- this.hydrated = true;
128468
- this.hydrateFailedAt = 0;
128469
- logger.info("PgCheckpointStore hydrated", {
128470
- rows: this.mirror.size
128471
- });
128472
- } catch (e) {
128473
- this.hydrateFailedAt = Date.now();
128474
- logger.warn("PgCheckpointStore hydrate failed (best-effort)", {
128475
- error: e.message
128476
- });
128477
- } finally {
128478
- this.hydrating = null;
128479
- }
128480
- })();
128481
- return this.hydrating;
128436
+ return MemoryClustering.instance;
128482
128437
  }
128483
- rowToCheckpoint(row) {
128484
- const state = decompressState(row.state);
128485
- const storedSchemaVersion = toNum4(row.state_schema_version);
128486
- if (storedSchemaVersion != null) {
128487
- const storedVersionString = String(storedSchemaVersion);
128488
- const normalized = /^\d+$/u.test(storedVersionString) ? `${storedVersionString}.0.0` : storedVersionString;
128489
- assertSchemaSupported("checkpoint", normalized, SUPPORTED_CHECKPOINT_STATE_SCHEMA_VERSION);
128438
+ constructor() {}
128439
+ async clusterMemories(k, maxIter = 20, maxMemories = 500) {
128440
+ const start = Date.now();
128441
+ const rows = await getPrismaClient2().$queryRaw`
128442
+ SELECT id, content, type, level, importance, array_to_json(tags)::text AS tags,
128443
+ embedding, created_at, updated_at, access_count, user_id, session_id, project_id, agent_id
128444
+ FROM memories WHERE embedding IS NOT NULL ORDER BY created_at DESC LIMIT ${maxMemories}`;
128445
+ const items = [];
128446
+ for (const row of rows) {
128447
+ if (!row.embedding)
128448
+ continue;
128449
+ const buf = row.embedding instanceof Buffer ? row.embedding : Buffer.from(row.embedding);
128450
+ const vec = Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
128451
+ if (vec.every((v) => v === 0))
128452
+ continue;
128453
+ items.push({ row, vec });
128490
128454
  }
128491
- return {
128492
- id: row.id,
128493
- taskId: row.task_id,
128494
- taskDescription: row.task_description ?? undefined,
128495
- agentId: row.agent_id ?? undefined,
128496
- projectId: row.project_id ?? undefined,
128497
- state,
128498
- memoryIds: row.memory_ids ? JSON.parse(row.memory_ids) : [],
128499
- fileChanges: row.file_changes ? JSON.parse(row.file_changes) : [],
128500
- checkpointType: row.checkpoint_type,
128501
- parentCheckpointId: row.parent_checkpoint_id ?? undefined,
128502
- createdAt: toNum4(row.created_at) ?? Date.now(),
128503
- expiresAt: toNum4(row.expires_at) ?? undefined
128504
- };
128505
- }
128506
- createCheckpoint(state, options = {}) {
128507
- const {
128508
- agentId,
128509
- projectId,
128510
- checkpointType = CheckpointType.MANUAL,
128511
- memoryIds = [],
128512
- fileChanges = [],
128513
- parentCheckpointId,
128514
- ttlMs = 7 * 24 * 60 * 60 * 1000
128515
- } = options;
128516
- const id = `ckpt_${checkpointType}_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`;
128517
- const now2 = Date.now();
128518
- const expiresAt = now2 + ttlMs;
128519
- const checkpoint = {
128520
- id,
128521
- taskId: state.taskId,
128522
- taskDescription: state.description,
128523
- agentId,
128524
- projectId,
128525
- state,
128526
- memoryIds,
128527
- fileChanges,
128528
- checkpointType,
128529
- parentCheckpointId,
128530
- createdAt: now2,
128531
- expiresAt
128532
- };
128533
- this.mirror.set(id, checkpoint);
128534
- this.ensureHydrated();
128535
- this.chainWrite(id, async () => {
128536
- const prisma2 = this.getClient();
128537
- const canonicalProjectId = projectId ? await getProjectIdentityAliasResolver().resolve(projectId) : projectId;
128538
- const compressed = compressState(state);
128539
- await prisma2.$executeRaw`
128540
- INSERT INTO task_checkpoints (
128541
- id, task_id, task_description, agent_id, project_id,
128542
- state, state_schema_version,
128543
- memory_ids, file_changes,
128544
- checkpoint_type, parent_checkpoint_id,
128545
- created_at, expires_at
128546
- ) VALUES (
128547
- ${id},
128548
- ${state.taskId},
128549
- ${state.description ?? null},
128550
- ${agentId ?? null},
128551
- ${canonicalProjectId ?? null},
128552
- ${compressed},
128553
- 1,
128554
- ${JSON.stringify(memoryIds)},
128555
- ${JSON.stringify(fileChanges)},
128556
- ${checkpointType},
128557
- ${parentCheckpointId ?? null},
128558
- ${now2}::bigint,
128559
- ${expiresAt}::bigint
128560
- )
128561
- ON CONFLICT (id) DO UPDATE SET
128562
- task_id = EXCLUDED.task_id,
128563
- task_description = EXCLUDED.task_description,
128564
- agent_id = EXCLUDED.agent_id,
128565
- project_id = EXCLUDED.project_id,
128566
- state = EXCLUDED.state,
128567
- state_schema_version = EXCLUDED.state_schema_version,
128568
- memory_ids = EXCLUDED.memory_ids,
128569
- file_changes = EXCLUDED.file_changes,
128570
- checkpoint_type = EXCLUDED.checkpoint_type,
128571
- parent_checkpoint_id = EXCLUDED.parent_checkpoint_id,
128572
- created_at = EXCLUDED.created_at,
128573
- expires_at = EXCLUDED.expires_at
128574
- `;
128575
- });
128576
- logger.info("Checkpoint created (PG)", {
128577
- id,
128578
- taskId: state.taskId,
128579
- type: checkpointType,
128580
- compressedBytes: compressState(state).byteLength
128455
+ if (items.length < 3) {
128456
+ return { clusters: [], unclustered: items.length, durationMs: Date.now() - start };
128457
+ }
128458
+ const effectiveK = k ?? Math.max(2, Math.min(20, Math.round(Math.sqrt(items.length / 2))));
128459
+ const dim = items[0].vec.length;
128460
+ let centroids = this.kMeansPlusPlusInit(items.map((i) => i.vec), effectiveK);
128461
+ let assignments = new Int32Array(items.length);
128462
+ for (let iter = 0;iter < maxIter; iter++) {
128463
+ const newAssignments = new Int32Array(items.length);
128464
+ for (let i = 0;i < items.length; i++) {
128465
+ let bestDist = Infinity;
128466
+ let bestCluster = 0;
128467
+ for (let c = 0;c < centroids.length; c++) {
128468
+ const dist = this.euclideanDistanceSq(items[i].vec, centroids[c]);
128469
+ if (dist < bestDist) {
128470
+ bestDist = dist;
128471
+ bestCluster = c;
128472
+ }
128473
+ }
128474
+ newAssignments[i] = bestCluster;
128475
+ }
128476
+ let changed = false;
128477
+ for (let i = 0;i < items.length; i++) {
128478
+ if (newAssignments[i] !== assignments[i]) {
128479
+ changed = true;
128480
+ break;
128481
+ }
128482
+ }
128483
+ assignments = newAssignments;
128484
+ if (!changed)
128485
+ break;
128486
+ const newCentroids = [];
128487
+ const counts = [];
128488
+ for (let c = 0;c < centroids.length; c++) {
128489
+ newCentroids.push(Array.from({ length: dim }, () => 0));
128490
+ counts.push(0);
128491
+ }
128492
+ for (let i = 0;i < items.length; i++) {
128493
+ const c = assignments[i];
128494
+ counts[c]++;
128495
+ for (let d = 0;d < dim; d++) {
128496
+ newCentroids[c][d] += items[i].vec[d];
128497
+ }
128498
+ }
128499
+ for (let c = 0;c < centroids.length; c++) {
128500
+ if (counts[c] === 0)
128501
+ continue;
128502
+ for (let d = 0;d < dim; d++) {
128503
+ newCentroids[c][d] /= counts[c];
128504
+ }
128505
+ }
128506
+ centroids = newCentroids;
128507
+ }
128508
+ const clusterMap = new Map;
128509
+ for (let i = 0;i < items.length; i++) {
128510
+ const c = assignments[i];
128511
+ if (!clusterMap.has(c))
128512
+ clusterMap.set(c, []);
128513
+ clusterMap.get(c).push(items[i].row);
128514
+ }
128515
+ const clusters = [];
128516
+ let unclustered = 0;
128517
+ for (const [cIdx, members] of clusterMap) {
128518
+ if (members.length < 2) {
128519
+ unclustered += members.length;
128520
+ continue;
128521
+ }
128522
+ const label = this.generateLabel(members);
128523
+ const avgImportance = members.reduce((sum, m) => sum + m.importance, 0) / members.length;
128524
+ const totalAccess = members.reduce((sum, m) => sum + m.access_count, 0);
128525
+ const dominantType = this.getDominantType(members);
128526
+ clusters.push({
128527
+ id: `cluster_${Date.now()}_${cIdx}`,
128528
+ centroid: centroids[cIdx],
128529
+ memberIds: members.map((m) => m.id),
128530
+ label,
128531
+ importance: Math.round(avgImportance * 100) / 100,
128532
+ totalAccess,
128533
+ dominantType
128534
+ });
128535
+ }
128536
+ clusters.sort((a, b) => b.importance - a.importance);
128537
+ logger.info("MemoryClustering: complete", {
128538
+ inputMemories: items.length,
128539
+ clusters: clusters.length,
128540
+ unclustered,
128541
+ k: effectiveK
128581
128542
  });
128582
- return checkpoint;
128583
- }
128584
- getCheckpoint(checkpointId) {
128585
- this.ensureHydrated();
128586
- return this.mirror.get(checkpointId) ?? null;
128587
- }
128588
- listCheckpoints(options = {}) {
128589
- this.ensureHydrated();
128590
- return this.applyFilters(this.mirror.values(), options);
128543
+ return { clusters, unclustered, durationMs: Date.now() - start };
128591
128544
  }
128592
- listCheckpointsMetadata(options = {}) {
128593
- this.ensureHydrated();
128594
- const filtered = this.applyFilters(this.mirror.values(), options);
128595
- return filtered.map((c) => this.checkpointToMetadata(c));
128545
+ async findCluster(memoryId, cached2) {
128546
+ const result = cached2 ?? await this.clusterMemories();
128547
+ for (const cluster of result.clusters) {
128548
+ if (cluster.memberIds.includes(memoryId)) {
128549
+ return cluster;
128550
+ }
128551
+ }
128552
+ return null;
128596
128553
  }
128597
- getCheckpointState(checkpointId) {
128598
- this.ensureHydrated();
128599
- const ckpt = this.mirror.get(checkpointId);
128600
- return ckpt ? ckpt.state : null;
128554
+ async summarizeCluster(cluster) {
128555
+ const members = await getPrismaClient2().$queryRaw`SELECT content, type, importance FROM memories WHERE id = ANY(${cluster.memberIds}::text[]) ORDER BY importance DESC LIMIT 5`;
128556
+ if (members.length === 0)
128557
+ return cluster.label;
128558
+ const lead = members[0].content.split(/[.!?\n]/)[0].trim();
128559
+ const typeCount = members.length;
128560
+ const types6 = [...new Set(members.map((m) => m.type))].join(", ");
128561
+ return `[${cluster.label}] ${lead} (${typeCount} memories, types: ${types6})`;
128601
128562
  }
128602
- getLatestCheckpoint(taskId) {
128603
- this.ensureHydrated();
128604
- const now2 = Date.now();
128605
- let latest = null;
128606
- for (const ckpt of this.mirror.values()) {
128607
- if (ckpt.taskId === taskId && (ckpt.expiresAt == null || ckpt.expiresAt > now2)) {
128608
- if (!latest || ckpt.createdAt > latest.createdAt) {
128609
- latest = ckpt;
128563
+ kMeansPlusPlusInit(vectors, k) {
128564
+ const centroids = [];
128565
+ const firstIdx = Math.floor(Math.random() * vectors.length);
128566
+ centroids.push([...vectors[firstIdx]]);
128567
+ for (let c = 1;c < k; c++) {
128568
+ const distances = [];
128569
+ let totalDist = 0;
128570
+ for (const vec of vectors) {
128571
+ let minDist = Infinity;
128572
+ for (const centroid of centroids) {
128573
+ const d = this.euclideanDistanceSq(vec, centroid);
128574
+ if (d < minDist)
128575
+ minDist = d;
128576
+ }
128577
+ distances.push(minDist);
128578
+ totalDist += minDist;
128579
+ }
128580
+ if (totalDist === 0) {
128581
+ centroids.push([...vectors[Math.floor(Math.random() * vectors.length)]]);
128582
+ continue;
128583
+ }
128584
+ let r = Math.random() * totalDist;
128585
+ for (let i = 0;i < distances.length; i++) {
128586
+ r -= distances[i];
128587
+ if (r <= 0) {
128588
+ centroids.push([...vectors[i]]);
128589
+ break;
128610
128590
  }
128611
128591
  }
128592
+ if (centroids.length <= c) {
128593
+ centroids.push([...vectors[vectors.length - 1]]);
128594
+ }
128595
+ }
128596
+ return centroids;
128597
+ }
128598
+ generateLabel(members) {
128599
+ const stopWords = new Set([
128600
+ "the",
128601
+ "a",
128602
+ "an",
128603
+ "is",
128604
+ "are",
128605
+ "was",
128606
+ "were",
128607
+ "be",
128608
+ "been",
128609
+ "being",
128610
+ "have",
128611
+ "has",
128612
+ "had",
128613
+ "do",
128614
+ "does",
128615
+ "did",
128616
+ "will",
128617
+ "would",
128618
+ "could",
128619
+ "should",
128620
+ "may",
128621
+ "might",
128622
+ "shall",
128623
+ "can",
128624
+ "need",
128625
+ "dare",
128626
+ "ought",
128627
+ "used",
128628
+ "to",
128629
+ "of",
128630
+ "in",
128631
+ "for",
128632
+ "on",
128633
+ "with",
128634
+ "at",
128635
+ "by",
128636
+ "from",
128637
+ "as",
128638
+ "into",
128639
+ "through",
128640
+ "during",
128641
+ "before",
128642
+ "after",
128643
+ "above",
128644
+ "below",
128645
+ "between",
128646
+ "out",
128647
+ "off",
128648
+ "over",
128649
+ "under",
128650
+ "again",
128651
+ "further",
128652
+ "then",
128653
+ "once",
128654
+ "here",
128655
+ "there",
128656
+ "when",
128657
+ "where",
128658
+ "why",
128659
+ "how",
128660
+ "all",
128661
+ "both",
128662
+ "each",
128663
+ "few",
128664
+ "more",
128665
+ "most",
128666
+ "other",
128667
+ "some",
128668
+ "such",
128669
+ "no",
128670
+ "nor",
128671
+ "not",
128672
+ "only",
128673
+ "own",
128674
+ "same",
128675
+ "so",
128676
+ "than",
128677
+ "too",
128678
+ "very",
128679
+ "just",
128680
+ "don",
128681
+ "should",
128682
+ "now",
128683
+ "and",
128684
+ "but",
128685
+ "or",
128686
+ "if",
128687
+ "while",
128688
+ "that",
128689
+ "this",
128690
+ "it",
128691
+ "its",
128692
+ "what",
128693
+ "which",
128694
+ "who",
128695
+ "whom",
128696
+ "these",
128697
+ "those",
128698
+ "i",
128699
+ "me",
128700
+ "my",
128701
+ "myself",
128702
+ "we",
128703
+ "our",
128704
+ "ours",
128705
+ "you",
128706
+ "your",
128707
+ "he",
128708
+ "him",
128709
+ "his",
128710
+ "she",
128711
+ "her",
128712
+ "they",
128713
+ "them",
128714
+ "their",
128715
+ "de",
128716
+ "da",
128717
+ "do",
128718
+ "das",
128719
+ "dos",
128720
+ "em",
128721
+ "no",
128722
+ "na",
128723
+ "nos",
128724
+ "nas",
128725
+ "um",
128726
+ "uma",
128727
+ "uns",
128728
+ "umas",
128729
+ "para",
128730
+ "com",
128731
+ "por",
128732
+ "que",
128733
+ "se",
128734
+ "como",
128735
+ "mas",
128736
+ "ou",
128737
+ "quando",
128738
+ "mais",
128739
+ "tamb\xE9m",
128740
+ "j\xE1",
128741
+ "ainda",
128742
+ "sobre",
128743
+ "entre",
128744
+ "at\xE9",
128745
+ "sem",
128746
+ "sob",
128747
+ "esse",
128748
+ "essa",
128749
+ "este",
128750
+ "esta",
128751
+ "aquele",
128752
+ "aquela",
128753
+ "ele",
128754
+ "ela",
128755
+ "eles",
128756
+ "elas",
128757
+ "n\xF3s",
128758
+ "eu",
128759
+ "tu",
128760
+ "voc\xEA",
128761
+ "voc\xEAs",
128762
+ "meu",
128763
+ "minha",
128764
+ "seu",
128765
+ "sua"
128766
+ ]);
128767
+ const wordFreq = new Map;
128768
+ for (const member of members) {
128769
+ const words = member.content.toLowerCase().replace(/[^a-z\u00E1\u00E0\u00E2\u00E3\u00E9\u00E8\u00EA\u00ED\u00EF\u00F3\u00F4\u00F5\u00FA\u00FC\u00E7\s-]/g, " ").split(/\s+/).filter((w) => w.length > 2 && !stopWords.has(w));
128770
+ const seen = new Set;
128771
+ for (const word of words) {
128772
+ if (seen.has(word))
128773
+ continue;
128774
+ seen.add(word);
128775
+ wordFreq.set(word, (wordFreq.get(word) ?? 0) + 1);
128776
+ }
128612
128777
  }
128613
- return latest;
128614
- }
128615
- deleteCheckpoint(checkpointId) {
128616
- const existed = this.mirror.has(checkpointId);
128617
- this.mirror.delete(checkpointId);
128618
- this.ensureHydrated();
128619
- this.chainWrite(checkpointId, async () => {
128620
- const prisma2 = this.getClient();
128621
- await prisma2.$executeRaw`
128622
- DELETE FROM task_checkpoints WHERE id = ${checkpointId}
128623
- `;
128624
- });
128625
- return existed;
128778
+ const sorted = [...wordFreq.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(([word]) => word);
128779
+ return sorted.length > 0 ? sorted.join(", ") : "misc";
128626
128780
  }
128627
- purgeExpired() {
128628
- const now2 = Date.now();
128629
- let count = 0;
128630
- const toRemove = [];
128631
- for (const ckpt of this.mirror.values()) {
128632
- if (ckpt.expiresAt != null && ckpt.expiresAt < now2) {
128633
- toRemove.push(ckpt.id);
128634
- count++;
128635
- }
128636
- }
128637
- for (const id of toRemove)
128638
- this.mirror.delete(id);
128639
- if (count > 0) {
128640
- this.chainWrite("__purge__", async () => {
128641
- const prisma2 = this.getClient();
128642
- await prisma2.$executeRaw`
128643
- DELETE FROM task_checkpoints
128644
- WHERE expires_at IS NOT NULL AND expires_at < ${now2}::bigint
128645
- `;
128646
- });
128647
- logger.info("Expired checkpoints purged (PG)", { count });
128781
+ getDominantType(members) {
128782
+ const typeCounts = new Map;
128783
+ for (const m of members) {
128784
+ typeCounts.set(m.type, (typeCounts.get(m.type) ?? 0) + 1);
128648
128785
  }
128649
- return count;
128650
- }
128651
- async countExistingMemoryIds(memoryIds) {
128652
- if (memoryIds.length === 0)
128653
- return [];
128654
- this.ensureHydrated();
128655
- const BATCH_SIZE = 1000;
128656
- const existing = [];
128657
- try {
128658
- const prisma2 = this.getClient();
128659
- for (let i = 0;i < memoryIds.length; i += BATCH_SIZE) {
128660
- const batch = memoryIds.slice(i, i + BATCH_SIZE);
128661
- const rows = await prisma2.$queryRaw`
128662
- SELECT id FROM memories WHERE id IN (${import_prisma4.Prisma.join(batch)})
128663
- `;
128664
- for (const row of rows)
128665
- existing.push(row.id);
128786
+ let dominant = "unknown";
128787
+ let maxCount = 0;
128788
+ for (const [type, count] of typeCounts) {
128789
+ if (count > maxCount) {
128790
+ maxCount = count;
128791
+ dominant = type;
128666
128792
  }
128667
- return existing;
128668
- } catch (e) {
128669
- logger.warn("countExistingMemoryIds failed (best-effort: assuming all exist)", {
128670
- error: e.message
128671
- });
128672
- return memoryIds;
128673
- }
128674
- }
128675
- getStats() {
128676
- this.ensureHydrated();
128677
- const checkpoints = Array.from(this.mirror.values());
128678
- const byType = {};
128679
- let totalSizeBytes = 0;
128680
- let oldest;
128681
- for (const c of checkpoints) {
128682
- byType[c.checkpointType] = (byType[c.checkpointType] ?? 0) + 1;
128683
- totalSizeBytes += compressState(c.state).byteLength;
128684
- if (oldest == null || c.createdAt < oldest)
128685
- oldest = c.createdAt;
128686
128793
  }
128687
- return {
128688
- totalCheckpoints: checkpoints.length,
128689
- byType,
128690
- totalSizeBytes,
128691
- oldestCheckpointAge: oldest != null ? Date.now() - oldest : undefined
128692
- };
128693
- }
128694
- ensureReady() {
128695
- return this.ensureHydrated();
128794
+ return dominant;
128696
128795
  }
128697
- close() {}
128698
- applyFilters(iter, options) {
128699
- const { taskId, projectId, checkpointType, includeExpired = false, limit = 20, offset = 0 } = options;
128700
- const now2 = Date.now();
128701
- const out = [];
128702
- for (const c of iter) {
128703
- if (taskId && c.taskId !== taskId)
128704
- continue;
128705
- if (projectId && c.projectId !== projectId)
128706
- continue;
128707
- if (checkpointType && c.checkpointType !== checkpointType)
128708
- continue;
128709
- if (!includeExpired && c.expiresAt != null && c.expiresAt <= now2)
128710
- continue;
128711
- out.push(c);
128796
+ euclideanDistanceSq(a, b) {
128797
+ let sum = 0;
128798
+ for (let i = 0;i < a.length; i++) {
128799
+ const diff = a[i] - b[i];
128800
+ sum += diff * diff;
128712
128801
  }
128713
- out.sort((a, b) => b.createdAt - a.createdAt);
128714
- return out.slice(offset, offset + limit);
128715
- }
128716
- checkpointToMetadata(c) {
128717
- return {
128718
- id: c.id,
128719
- taskId: c.taskId,
128720
- taskDescription: c.taskDescription,
128721
- agentId: c.agentId,
128722
- projectId: c.projectId,
128723
- checkpointType: c.checkpointType,
128724
- parentCheckpointId: c.parentCheckpointId,
128725
- createdAt: c.createdAt,
128726
- expiresAt: c.expiresAt,
128727
- compressedSizeBytes: compressState(c.state).byteLength,
128728
- memoryCount: c.memoryIds.length,
128729
- fileChangeCount: c.fileChanges.length
128730
- };
128731
- }
128732
- chainWrite(key, fn) {
128733
- const prev = this.inflight.get(key) ?? Promise.resolve();
128734
- const next = prev.then(fn).catch((e) => {
128735
- logger.warn("PgCheckpointStore write failed (best-effort)", {
128736
- key,
128737
- error: e.message
128738
- });
128739
- });
128740
- this.inflight.set(key, next);
128741
- next.then(() => {
128742
- if (this.inflight.get(key) === next)
128743
- this.inflight.delete(key);
128744
- });
128745
- }
128746
- async __drain() {
128747
- const pending = Array.from(this.inflight.values());
128748
- if (pending.length > 0)
128749
- await Promise.allSettled(pending);
128750
- await new Promise((r) => setTimeout(r, 10));
128802
+ return sum;
128751
128803
  }
128752
- async __hydrate() {
128753
- await this.ensureHydrated();
128804
+ close() {
128805
+ MemoryClustering.instance = null;
128754
128806
  }
128755
128807
  }
128756
- var import_prisma4, SUPPORTED_CHECKPOINT_STATE_SCHEMA_VERSION = "1.0.0";
128757
- var init_checkpoint_store_pg = __esm(() => {
128808
+ var init_memory_clustering = __esm(() => {
128758
128809
  init_dist();
128759
128810
  init_prisma_client();
128760
- init_alias_resolver();
128761
- init_schema_version();
128762
- import_prisma4 = __toESM(require_prisma(), 1);
128763
- });
128764
-
128765
- // ../../packages/core/dist/services/checkpoint/checkpoint-manager.js
128766
- var CheckpointManager;
128767
- var init_checkpoint_manager = __esm(() => {
128768
- init_config();
128769
- init_checkpoint_store_pg();
128770
- CheckpointManager = class CheckpointManager extends PgCheckpointStore {
128771
- static instance = null;
128772
- static getInstance() {
128773
- requirePostgresDatabaseUrl();
128774
- return this.instance ??= new CheckpointManager;
128775
- }
128776
- async restoreCheckpoint(checkpointId) {
128777
- const checkpoint = this.getCheckpoint(checkpointId);
128778
- if (!checkpoint)
128779
- return null;
128780
- const existing = new Set(await this.countExistingMemoryIds(checkpoint.memoryIds));
128781
- const validMemoryIds = checkpoint.memoryIds.filter((id) => existing.has(id));
128782
- const missingMemoryIds = checkpoint.memoryIds.filter((id) => !existing.has(id));
128783
- const fileConflicts = [];
128784
- const restoreInstructions = [
128785
- `Restore checkpoint ${checkpoint.id} for task ${checkpoint.taskId}.`,
128786
- missingMemoryIds.length ? `Missing memories: ${missingMemoryIds.join(", ")}.` : "All referenced memories are available."
128787
- ].join(`
128788
- `);
128789
- return { checkpoint, validMemoryIds, missingMemoryIds, fileConflicts, restoreInstructions };
128790
- }
128791
- };
128792
128811
  });
128793
128812
 
128794
128813
  // ../../packages/core/dist/services/checkpoint/auto-checkpointer.js