@martian-engineering/lossless-claw 0.13.1 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -16
- package/dist/index.js +86 -36
- package/dist/migrate-sessions.js +941 -0
- package/docs/configuration.md +19 -11
- package/openclaw.plugin.json +18 -0
- package/package.json +11 -8
- package/skills/lossless-claw/SKILL.md +4 -3
- package/skills/lossless-claw/references/architecture.md +3 -3
- package/skills/lossless-claw/references/config.md +21 -2
- package/skills/lossless-claw/references/session-lifecycle.md +34 -0
|
@@ -0,0 +1,941 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import{constants}from"node:fs";import{access,mkdir,readdir,stat}from"node:fs/promises";import{basename,dirname as dirname2,extname,join,resolve as resolve2}from"node:path";import{homedir}from"node:os";import{DatabaseSync as DatabaseSync2}from"node:sqlite";import{mkdirSync}from"node:fs";import{dirname,resolve}from"node:path";import{DatabaseSync}from"node:sqlite";var SQLITE_BUSY_TIMEOUT_MS=3e4;var connectionsByPath=new Map;var connectionIndex=new Map;function normalizeDbPathInput(dbPath){return typeof dbPath==="string"?dbPath.trim():""}function isInMemoryPath(dbPath){const normalized=normalizeDbPathInput(dbPath);return normalized===":memory:"||normalized.startsWith("file::memory:")}function getFileBackedDatabasePath(dbPath){const trimmed=normalizeDbPathInput(dbPath);if(!trimmed||isInMemoryPath(trimmed)){return null}return resolve(trimmed)}function normalizePath(dbPath){const fileBackedDatabasePath=getFileBackedDatabasePath(dbPath);if(!fileBackedDatabasePath){const trimmed=normalizeDbPathInput(dbPath);return trimmed.length>0?trimmed:":memory:"}return fileBackedDatabasePath}function ensureDbDirectory(dbPath){const fileBackedDatabasePath=getFileBackedDatabasePath(dbPath);if(!fileBackedDatabasePath){return}mkdirSync(dirname(fileBackedDatabasePath),{recursive:true})}function configureConnection(db){db.exec("PRAGMA journal_mode = WAL");db.exec(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`);db.exec("PRAGMA foreign_keys = ON");db.exec("PRAGMA cache_size = -65536");db.exec("PRAGMA synchronous = NORMAL");db.exec("PRAGMA temp_store = MEMORY");if(typeof db.enableLoadExtension==="function"){db.enableLoadExtension(false)}return db}function createDatabaseSync(dbPath){const supportsExtensionLoading=typeof DatabaseSync.prototype.enableLoadExtension==="function";return supportsExtensionLoading?new DatabaseSync(dbPath,{allowExtension:true}):new DatabaseSync(dbPath)}function trackConnection(dbPath,db){const key=normalizePath(dbPath);let entries=connectionsByPath.get(key);if(!entries){entries=new Set;connectionsByPath.set(key,entries)}entries.add(db);connectionIndex.set(db,key)}function untrackConnection(db){const key=connectionIndex.get(db);if(!key){return}const entries=connectionsByPath.get(key);if(entries){entries.delete(db);if(entries.size===0){connectionsByPath.delete(key)}}connectionIndex.delete(db)}function closeDatabase(db){if(!db){return}try{try{db.exec("PRAGMA optimize")}catch{}db.close()}catch{}finally{untrackConnection(db)}}function createLcmDatabaseConnection(dbPath){ensureDbDirectory(dbPath);const db=createDatabaseSync(dbPath);try{configureConnection(db)}catch(err){try{db.close()}catch{}throw err}trackConnection(dbPath,db);return db}function closeLcmConnection(target){if(target&&typeof target!=="string"){closeDatabase(target);return}if(typeof target==="string"){const key=normalizePath(target);const entries=connectionsByPath.get(key);if(!entries){return}for(const db of[...entries]){closeDatabase(db)}connectionsByPath.delete(key);return}for(const db of[...connectionIndex.keys()]){closeDatabase(db)}connectionsByPath.clear();connectionIndex.clear()}import{createHash as createHash2}from"node:crypto";var featureCache=new WeakMap;function probeVirtualTable(db,sql){try{db.exec("DROP TABLE IF EXISTS temp.__lcm_virtual_table_probe");db.exec(sql);db.exec("DROP TABLE temp.__lcm_virtual_table_probe");return true}catch{try{db.exec("DROP TABLE IF EXISTS temp.__lcm_virtual_table_probe")}catch{}return false}}function probeFts5(db){return probeVirtualTable(db,"CREATE VIRTUAL TABLE temp.__lcm_virtual_table_probe USING fts5(content)")}function probeTrigramTokenizer(db){return probeVirtualTable(db,"CREATE VIRTUAL TABLE temp.__lcm_virtual_table_probe USING fts5(content, tokenize='trigram')")}function getLcmDbFeatures(db){const cached=featureCache.get(db);if(cached){return cached}const detected={fts5Available:probeFts5(db),trigramTokenizerAvailable:false};if(detected.fts5Available){detected.trigramTokenizerAvailable=probeTrigramTokenizer(db)}featureCache.set(db,detected);return detected}var OPENCLAW_INBOUND_METADATA_BLOCK_RE=/^(Conversation info \(untrusted metadata\)|Sender \(untrusted metadata\)):\r?\n```json\r?\n([\s\S]*?)\r?\n```/;var CONVERSATION_INFO_KEYS=new Set(["chat_id","message_id","reply_to_id","sender_id","conversation_label","sender","timestamp","group_subject","group_channel","group_space","group_members","thread_label","inbound_event_kind","topic_id","topic_name","is_forum","mention_reason","mention_target","mentioned_user_ids","mentioned_usernames","has_reply_context","has_forwarded_context","has_thread_starter","history_count","history_media_count","history_truncated"]);var VOLATILE_CONVERSATION_INFO_KEYS=new Set(["message_id","reply_to_id","timestamp"]);var SENDER_INFO_KEYS=new Set(["label","id","name","username","tag","e164"]);function canonicalizeOpenClawInboundMetadataIdentityContent(role,content){if(role!=="user"){return content}const{prelude,metadataCandidate}=splitOpenClawInboundMetadataPrelude(content);const conversationCandidate=metadataCandidate.trimStart();const conversationMatch=OPENCLAW_INBOUND_METADATA_BLOCK_RE.exec(conversationCandidate);const conversationHeading=conversationMatch?.[1]??"";const conversationRecord=conversationMatch?parseOpenClawInboundMetadataRecord(conversationHeading,conversationMatch[2]??""):null;const canonicalConversationJson=conversationRecord?canonicalizeMetadataJson(conversationRecord,VOLATILE_CONVERSATION_INFO_KEYS):null;if(!conversationMatch||conversationHeading!=="Conversation info (untrusted metadata)"||!canonicalConversationJson){return content}let remaining=conversationCandidate.slice(conversationMatch[0].length);const canonicalBlocks=[formatCanonicalMetadataBlock(conversationHeading,canonicalConversationJson)];const senderCandidate=remaining.trimStart();const senderMatch=OPENCLAW_INBOUND_METADATA_BLOCK_RE.exec(senderCandidate);const senderHeading=senderMatch?.[1]??"";const senderRecord=senderMatch?parseOpenClawInboundMetadataRecord(senderHeading,senderMatch[2]??""):null;const canonicalSenderJson=senderRecord?canonicalizeMetadataJson(senderRecord,new Set):null;if(senderMatch&&senderHeading==="Sender (untrusted metadata)"&&canonicalSenderJson){remaining=stripMetadataSeparator(senderCandidate.slice(senderMatch[0].length));canonicalBlocks.push(formatCanonicalMetadataBlock(senderHeading,canonicalSenderJson))}else{remaining=stripMetadataSeparator(remaining)}return remaining.trim().length>0?`${prelude}${canonicalBlocks.join("\n\n")}
|
|
3
|
+
|
|
4
|
+
${remaining}`:content}function splitOpenClawInboundMetadataPrelude(content){const trimmed=content.trimStart();if(trimmed.startsWith("Conversation info (untrusted metadata):")){return{prelude:"",metadataCandidate:trimmed}}const deliveryPrelude=/^Delivery:[\s\S]*?\r?\n\r?\n(?=Conversation info \(untrusted metadata\):)/.exec(trimmed);if(!deliveryPrelude){return{prelude:"",metadataCandidate:trimmed}}return{prelude:deliveryPrelude[0],metadataCandidate:trimmed.slice(deliveryPrelude[0].length)}}function parseOpenClawInboundMetadataRecord(heading,json){let parsed;try{parsed=JSON.parse(json)}catch{return null}if(!parsed||typeof parsed!=="object"||Array.isArray(parsed)){return null}const knownKeys=getKnownKeysForHeading(heading);if(!knownKeys){return null}return Object.keys(parsed).some(key=>knownKeys.has(key))?parsed:null}function canonicalizeMetadataJson(record,volatileKeys){const stableEntries=Object.entries(record).filter(([key])=>!volatileKeys.has(key)).map(([key,value])=>[key,canonicalizeJsonValue(value)]).sort(([left],[right])=>left.localeCompare(right));if(stableEntries.length===0){return null}return JSON.stringify(Object.fromEntries(stableEntries))}function canonicalizeJsonValue(value){if(Array.isArray(value)){return value.map(item=>canonicalizeJsonValue(item))}if(!value||typeof value!=="object"){return value}return Object.fromEntries(Object.entries(value).map(([key,nestedValue])=>[key,canonicalizeJsonValue(nestedValue)]).sort(([left],[right])=>left.localeCompare(right)))}function formatCanonicalMetadataBlock(heading,json){return[heading+":","```json",json,"```"].join("\n")}function stripMetadataSeparator(content){return content.replace(/^[ \t]*(?:\r?\n)(?:[ \t]*(?:\r?\n))?/,"")}function getKnownKeysForHeading(heading){if(heading==="Conversation info (untrusted metadata)"){return CONVERSATION_INFO_KEYS}if(heading==="Sender (untrusted metadata)"){return SENDER_INFO_KEYS}return void 0}import{createHash}from"node:crypto";function buildMessageIdentityHash(role,content){const identityContent=canonicalizeOpenClawInboundMetadataIdentityContent(role,content);return createHash("sha256").update(role).update("\0").update(identityContent).digest("hex")}function parseUtcTimestamp(value){if(typeof value!=="string")return new Date(Number.NaN);const s=value.trim();if(/(?:[zZ]|[+-]\d{2}:\d{2})$/.test(s)){return new Date(s)}const normalized=s.includes("T")?s:s.replace(" ","T");return new Date(`${normalized}Z`)}function parseUtcTimestampOrNull(value){if(value==null)return null;return parseUtcTimestamp(value)}var VERSIONED_BACKFILL_STEPS={backfillSummaryDepths:1,backfillSummaryMetadata:1,backfillToolCallColumns:1,repairOpenClawMetadataIdentityState:1};function ensureSummaryDepthColumn(db){const summaryColumns=db.prepare(`PRAGMA table_info(summaries)`).all();const hasDepth=summaryColumns.some(col=>col.name==="depth");if(!hasDepth){db.exec(`ALTER TABLE summaries ADD COLUMN depth INTEGER NOT NULL DEFAULT 0`)}}function ensureSummaryMetadataColumns(db){const summaryColumns=db.prepare(`PRAGMA table_info(summaries)`).all();const hasEarliestAt=summaryColumns.some(col=>col.name==="earliest_at");const hasLatestAt=summaryColumns.some(col=>col.name==="latest_at");const hasDescendantCount=summaryColumns.some(col=>col.name==="descendant_count");const hasDescendantTokenCount=summaryColumns.some(col=>col.name==="descendant_token_count");const hasSourceMessageTokenCount=summaryColumns.some(col=>col.name==="source_message_token_count");if(!hasEarliestAt){db.exec(`ALTER TABLE summaries ADD COLUMN earliest_at TEXT`)}if(!hasLatestAt){db.exec(`ALTER TABLE summaries ADD COLUMN latest_at TEXT`)}if(!hasDescendantCount){db.exec(`ALTER TABLE summaries ADD COLUMN descendant_count INTEGER NOT NULL DEFAULT 0`)}if(!hasDescendantTokenCount){db.exec(`ALTER TABLE summaries ADD COLUMN descendant_token_count INTEGER NOT NULL DEFAULT 0`)}if(!hasSourceMessageTokenCount){db.exec(`ALTER TABLE summaries ADD COLUMN source_message_token_count INTEGER NOT NULL DEFAULT 0`)}}function parseTimestamp(value){return parseUtcTimestampOrNull(value)}function isoStringOrNull(value){return value?value.toISOString():null}function ensureSummaryModelColumn(db){const summaryColumns=db.prepare(`PRAGMA table_info(summaries)`).all();const hasModel=summaryColumns.some(col=>col.name==="model");if(!hasModel){db.exec(`ALTER TABLE summaries ADD COLUMN model TEXT NOT NULL DEFAULT 'unknown'`)}}function ensureCompactionTelemetryColumns(db){const telemetryColumns=db.prepare(`PRAGMA table_info(conversation_compaction_telemetry)`).all();const hasConsecutiveColdObservations=telemetryColumns.some(col=>col.name==="consecutive_cold_observations");const hasLastLeafCompactionAt=telemetryColumns.some(col=>col.name==="last_leaf_compaction_at");const hasTurnsSinceLeafCompaction=telemetryColumns.some(col=>col.name==="turns_since_leaf_compaction");const hasTokensAccumulatedSinceLeafCompaction=telemetryColumns.some(col=>col.name==="tokens_accumulated_since_leaf_compaction");const hasLastActivityBand=telemetryColumns.some(col=>col.name==="last_activity_band");const hasLastApiCallAt=telemetryColumns.some(col=>col.name==="last_api_call_at");const hasLastCacheTouchAt=telemetryColumns.some(col=>col.name==="last_cache_touch_at");const hasProvider=telemetryColumns.some(col=>col.name==="provider");const hasModel=telemetryColumns.some(col=>col.name==="model");const hasLastObservedPromptTokenCount=telemetryColumns.some(col=>col.name==="last_observed_prompt_token_count");if(!hasConsecutiveColdObservations){db.exec(`ALTER TABLE conversation_compaction_telemetry ADD COLUMN consecutive_cold_observations INTEGER NOT NULL DEFAULT 0`)}if(!hasLastLeafCompactionAt){db.exec(`ALTER TABLE conversation_compaction_telemetry ADD COLUMN last_leaf_compaction_at TEXT`)}if(!hasTurnsSinceLeafCompaction){db.exec(`ALTER TABLE conversation_compaction_telemetry ADD COLUMN turns_since_leaf_compaction INTEGER NOT NULL DEFAULT 0`)}if(!hasTokensAccumulatedSinceLeafCompaction){db.exec(`ALTER TABLE conversation_compaction_telemetry ADD COLUMN tokens_accumulated_since_leaf_compaction INTEGER NOT NULL DEFAULT 0`)}if(!hasLastActivityBand){db.exec(`ALTER TABLE conversation_compaction_telemetry ADD COLUMN last_activity_band TEXT NOT NULL DEFAULT 'low' CHECK (last_activity_band IN ('low', 'medium', 'high'))`)}if(!hasLastApiCallAt){db.exec(`ALTER TABLE conversation_compaction_telemetry ADD COLUMN last_api_call_at TEXT`)}if(!hasLastCacheTouchAt){db.exec(`ALTER TABLE conversation_compaction_telemetry ADD COLUMN last_cache_touch_at TEXT`)}if(!hasProvider){db.exec(`ALTER TABLE conversation_compaction_telemetry ADD COLUMN provider TEXT`)}if(!hasModel){db.exec(`ALTER TABLE conversation_compaction_telemetry ADD COLUMN model TEXT`)}if(!hasLastObservedPromptTokenCount){db.exec(`ALTER TABLE conversation_compaction_telemetry ADD COLUMN last_observed_prompt_token_count INTEGER`)}}function ensureCompactionMaintenanceColumns(db){const maintenanceColumns=db.prepare(`PRAGMA table_info(conversation_compaction_maintenance)`).all();const hasProjectedTokenCount=maintenanceColumns.some(col=>col.name==="projected_token_count");const hasRawTokensOutsideTail=maintenanceColumns.some(col=>col.name==="raw_tokens_outside_tail");const hasRetryAttempts=maintenanceColumns.some(col=>col.name==="retry_attempts");const hasNextAttemptAfter=maintenanceColumns.some(col=>col.name==="next_attempt_after");const hasContextThreshold=maintenanceColumns.some(col=>col.name==="context_threshold");const hasContextThresholdSource=maintenanceColumns.some(col=>col.name==="context_threshold_source");if(!hasProjectedTokenCount){db.exec(`ALTER TABLE conversation_compaction_maintenance ADD COLUMN projected_token_count INTEGER`)}if(!hasRawTokensOutsideTail){db.exec(`ALTER TABLE conversation_compaction_maintenance ADD COLUMN raw_tokens_outside_tail INTEGER`)}if(!hasRetryAttempts){db.exec(`ALTER TABLE conversation_compaction_maintenance ADD COLUMN retry_attempts INTEGER NOT NULL DEFAULT 0`)}if(!hasNextAttemptAfter){db.exec(`ALTER TABLE conversation_compaction_maintenance ADD COLUMN next_attempt_after TEXT`)}if(!hasContextThreshold){db.exec(`ALTER TABLE conversation_compaction_maintenance ADD COLUMN context_threshold REAL`)}if(!hasContextThresholdSource){db.exec(`ALTER TABLE conversation_compaction_maintenance ADD COLUMN context_threshold_source TEXT`)}}function ensureFocusBriefTables(db){db.exec(`
|
|
5
|
+
CREATE TABLE IF NOT EXISTS focus_briefs (
|
|
6
|
+
brief_id TEXT PRIMARY KEY,
|
|
7
|
+
conversation_id INTEGER NOT NULL REFERENCES conversations(conversation_id) ON DELETE CASCADE,
|
|
8
|
+
session_key TEXT,
|
|
9
|
+
prompt TEXT NOT NULL,
|
|
10
|
+
content TEXT NOT NULL,
|
|
11
|
+
status TEXT NOT NULL CHECK (status IN ('draft', 'active', 'superseded', 'failed', 'inactive')),
|
|
12
|
+
token_count INTEGER NOT NULL DEFAULT 0,
|
|
13
|
+
target_tokens INTEGER NOT NULL DEFAULT 0,
|
|
14
|
+
covered_latest_at TEXT,
|
|
15
|
+
covered_message_seq INTEGER,
|
|
16
|
+
source_context_hash TEXT NOT NULL DEFAULT '',
|
|
17
|
+
generator_run_id TEXT,
|
|
18
|
+
generator_session_key TEXT,
|
|
19
|
+
raw_result_json TEXT,
|
|
20
|
+
error TEXT,
|
|
21
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
22
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
23
|
+
superseded_at TEXT
|
|
24
|
+
);
|
|
25
|
+
|
|
26
|
+
CREATE TABLE IF NOT EXISTS focus_brief_sources (
|
|
27
|
+
brief_id TEXT NOT NULL REFERENCES focus_briefs(brief_id) ON DELETE CASCADE,
|
|
28
|
+
summary_id TEXT NOT NULL,
|
|
29
|
+
ordinal INTEGER,
|
|
30
|
+
role TEXT NOT NULL,
|
|
31
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
32
|
+
PRIMARY KEY (brief_id, summary_id, role)
|
|
33
|
+
);
|
|
34
|
+
|
|
35
|
+
CREATE INDEX IF NOT EXISTS focus_briefs_conversation_status_idx
|
|
36
|
+
ON focus_briefs (conversation_id, status, created_at);
|
|
37
|
+
CREATE INDEX IF NOT EXISTS focus_brief_sources_summary_idx
|
|
38
|
+
ON focus_brief_sources (summary_id);
|
|
39
|
+
`)}function ensureMessagePartsTable(db){const tables=db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'message_parts'`).all();if(tables.length>0)return;db.exec(`
|
|
40
|
+
CREATE TABLE IF NOT EXISTS message_parts (
|
|
41
|
+
part_id TEXT PRIMARY KEY,
|
|
42
|
+
message_id INTEGER NOT NULL REFERENCES messages(message_id) ON DELETE CASCADE,
|
|
43
|
+
session_id TEXT NOT NULL,
|
|
44
|
+
part_type TEXT NOT NULL CHECK (part_type IN (
|
|
45
|
+
'text', 'reasoning', 'tool', 'patch', 'file',
|
|
46
|
+
'subtask', 'compaction', 'step_start', 'step_finish',
|
|
47
|
+
'snapshot', 'agent', 'retry'
|
|
48
|
+
)),
|
|
49
|
+
ordinal INTEGER NOT NULL,
|
|
50
|
+
text_content TEXT,
|
|
51
|
+
is_ignored INTEGER,
|
|
52
|
+
is_synthetic INTEGER,
|
|
53
|
+
tool_call_id TEXT,
|
|
54
|
+
tool_name TEXT,
|
|
55
|
+
tool_status TEXT,
|
|
56
|
+
tool_input TEXT,
|
|
57
|
+
tool_output TEXT,
|
|
58
|
+
tool_error TEXT,
|
|
59
|
+
tool_title TEXT,
|
|
60
|
+
patch_hash TEXT,
|
|
61
|
+
patch_files TEXT,
|
|
62
|
+
file_mime TEXT,
|
|
63
|
+
file_name TEXT,
|
|
64
|
+
file_url TEXT,
|
|
65
|
+
subtask_prompt TEXT,
|
|
66
|
+
subtask_desc TEXT,
|
|
67
|
+
subtask_agent TEXT,
|
|
68
|
+
step_reason TEXT,
|
|
69
|
+
step_cost REAL,
|
|
70
|
+
step_tokens_in INTEGER,
|
|
71
|
+
step_tokens_out INTEGER,
|
|
72
|
+
snapshot_hash TEXT,
|
|
73
|
+
compaction_auto INTEGER,
|
|
74
|
+
metadata TEXT,
|
|
75
|
+
UNIQUE (message_id, ordinal)
|
|
76
|
+
)
|
|
77
|
+
`);db.exec(`CREATE INDEX IF NOT EXISTS message_parts_message_idx ON message_parts (message_id)`);db.exec(`CREATE INDEX IF NOT EXISTS message_parts_type_idx ON message_parts (part_type)`)}function ensureMessageIdentityHashColumn(db){const messageColumns=db.prepare(`PRAGMA table_info(messages)`).all();const hasIdentityHash=messageColumns.some(col=>col.name==="identity_hash");if(!hasIdentityHash){db.exec(`ALTER TABLE messages ADD COLUMN identity_hash TEXT`)}}function ensureMessageLargeContentColumn(db){const cols=db.prepare(`PRAGMA table_info(messages)`).all();const hasLargeContent=cols.some(c=>c.name==="large_content");if(!hasLargeContent){db.exec(`ALTER TABLE messages ADD COLUMN large_content TEXT`)}}function ensureMessageTranscriptEntryIdColumn(db){const messageColumns=db.prepare(`PRAGMA table_info(messages)`).all();const hasTranscriptEntryId=messageColumns.some(col=>col.name==="transcript_entry_id");if(!hasTranscriptEntryId){db.exec(`ALTER TABLE messages ADD COLUMN transcript_entry_id TEXT`)}}function ensureConversationBootstrapStateForkColumns(db){const columns=db.prepare(`PRAGMA table_info(conversation_bootstrap_state)`).all();const hasForkBounded=columns.some(col=>col.name==="fork_bounded");const hasForkSourceMessageCount=columns.some(col=>col.name==="fork_source_message_count");if(!hasForkBounded){db.exec(`ALTER TABLE conversation_bootstrap_state ADD COLUMN fork_bounded INTEGER NOT NULL DEFAULT 0`)}if(!hasForkSourceMessageCount){db.exec(`ALTER TABLE conversation_bootstrap_state ADD COLUMN fork_source_message_count INTEGER NOT NULL DEFAULT 0`)}}function ensureConversationBootstrapStateEpochColumns(db){const columns=db.prepare(`PRAGMA table_info(conversation_bootstrap_state)`).all();if(!columns.some(col=>col.name==="session_header_id")){db.exec(`ALTER TABLE conversation_bootstrap_state ADD COLUMN session_header_id TEXT`)}if(!columns.some(col=>col.name==="last_processed_entry_id")){db.exec(`ALTER TABLE conversation_bootstrap_state ADD COLUMN last_processed_entry_id TEXT`)}}function backfillMessageIdentityHashes(db,options){const selectStmt=db.prepare(`SELECT message_id, role, content
|
|
78
|
+
FROM messages
|
|
79
|
+
WHERE message_id > ?
|
|
80
|
+
AND (identity_hash IS NULL OR identity_hash = '')
|
|
81
|
+
ORDER BY message_id
|
|
82
|
+
LIMIT ?`);const updateStmt=db.prepare(`UPDATE messages SET identity_hash = ? WHERE message_id = ?`);let lastProcessedMessageId=0;const managesOwnTransaction=options?.managesOwnTransaction??true;while(true){const rows=selectStmt.all(lastProcessedMessageId,1e3);if(rows.length===0){return}if(managesOwnTransaction){db.exec(`BEGIN`)}try{for(const row of rows){updateStmt.run(buildMessageIdentityHash(row.role,row.content),row.message_id)}if(managesOwnTransaction){db.exec(`COMMIT`)}}catch(error){if(managesOwnTransaction){try{db.exec(`ROLLBACK`)}catch{}}throw error}lastProcessedMessageId=rows[rows.length-1]?.message_id??lastProcessedMessageId}}function buildLegacyRawMessageIdentityHash(role,content){return createHash2("sha256").update(role).update("\0").update(content).digest("hex")}function buildBootstrapEntryHash(role,content){return createHash2("sha256").update(JSON.stringify({role,content})).digest("hex")}function buildCanonicalBootstrapEntryHash(role,content){return buildBootstrapEntryHash(role,canonicalizeOpenClawInboundMetadataIdentityContent(role,content))}function repairOpenClawMetadataIdentityState(db){const selectStmt=db.prepare(`SELECT message_id, conversation_id, role, content, identity_hash
|
|
83
|
+
FROM messages
|
|
84
|
+
WHERE message_id > ? AND role = 'user'
|
|
85
|
+
ORDER BY message_id
|
|
86
|
+
LIMIT ?`);const updateIdentityStmt=db.prepare(`UPDATE messages SET identity_hash = ? WHERE message_id = ?`);const updateCheckpointStmt=db.prepare(`UPDATE conversation_bootstrap_state
|
|
87
|
+
SET last_processed_entry_hash = ?
|
|
88
|
+
WHERE conversation_id = ? AND last_processed_entry_hash = ?`);let lastProcessedMessageId=0;while(true){const rows=selectStmt.all(lastProcessedMessageId,1e3);if(rows.length===0){return}for(const row of rows){const canonicalContent=canonicalizeOpenClawInboundMetadataIdentityContent(row.role,row.content);if(canonicalContent===row.content){continue}const legacyMessageHash=buildLegacyRawMessageIdentityHash(row.role,row.content);if(row.identity_hash===legacyMessageHash){updateIdentityStmt.run(buildMessageIdentityHash(row.role,row.content),row.message_id)}updateCheckpointStmt.run(buildCanonicalBootstrapEntryHash(row.role,row.content),row.conversation_id,buildBootstrapEntryHash(row.role,row.content))}lastProcessedMessageId=rows[rows.length-1]?.message_id??lastProcessedMessageId}}function describeMigrationError(error){return error instanceof Error?error.message:String(error)}function runMigrationStep(name,log,step){const startedAt=Date.now();try{step();log?.info?.(`[lcm] migration step complete: step=${name} durationMs=${Date.now()-startedAt}`)}catch(error){log?.info?.(`[lcm] migration step failed: step=${name} durationMs=${Date.now()-startedAt} error=${describeMigrationError(error)}`);throw error}}function getVersionedBackfillSavepointName(stepName){return`lcm_backfill_${stepName}`}function hasCompletedVersionedBackfill(db,stepName,algorithmVersion){const row=db.prepare(`SELECT 1
|
|
89
|
+
FROM lcm_migration_state
|
|
90
|
+
WHERE step_name = ? AND algorithm_version = ?
|
|
91
|
+
LIMIT 1`).get(stepName,algorithmVersion);return row!=null}function markVersionedBackfillComplete(db,stepName,algorithmVersion){db.prepare(`INSERT INTO lcm_migration_state (step_name, algorithm_version, completed_at)
|
|
92
|
+
VALUES (?, ?, datetime('now'))
|
|
93
|
+
ON CONFLICT(step_name, algorithm_version)
|
|
94
|
+
DO UPDATE SET completed_at = excluded.completed_at`).run(stepName,algorithmVersion)}function rollbackSavepoint(db,savepointName){try{db.exec(`ROLLBACK TO SAVEPOINT ${savepointName}`)}finally{db.exec(`RELEASE SAVEPOINT ${savepointName}`)}}function runVersionedBackfillStep(db,stepName,log,step){const algorithmVersion=VERSIONED_BACKFILL_STEPS[stepName];if(hasCompletedVersionedBackfill(db,stepName,algorithmVersion)){log?.info?.(`[lcm] migration step skipped: step=${stepName} algorithmVersion=${algorithmVersion} reason=already-complete`);return}const startedAt=Date.now();const savepointName=getVersionedBackfillSavepointName(stepName);db.exec(`SAVEPOINT ${savepointName}`);try{step();markVersionedBackfillComplete(db,stepName,algorithmVersion);db.exec(`RELEASE SAVEPOINT ${savepointName}`);log?.info?.(`[lcm] migration step complete: step=${stepName} algorithmVersion=${algorithmVersion} durationMs=${Date.now()-startedAt}`)}catch(error){rollbackSavepoint(db,savepointName);log?.info?.(`[lcm] migration step failed: step=${stepName} algorithmVersion=${algorithmVersion} durationMs=${Date.now()-startedAt} error=${describeMigrationError(error)}`);throw error}}function backfillSummaryDepths(db){db.exec(`UPDATE summaries SET depth = 0 WHERE kind = 'leaf'`);const conversationRows=db.prepare(`SELECT DISTINCT conversation_id FROM summaries WHERE kind = 'condensed'`).all();if(conversationRows.length===0){return}const updateDepthStmt=db.prepare(`UPDATE summaries SET depth = ? WHERE summary_id = ?`);for(const row of conversationRows){const conversationId=row.conversation_id;const summaries=db.prepare(`SELECT summary_id, conversation_id, kind, depth, token_count, created_at
|
|
95
|
+
FROM summaries
|
|
96
|
+
WHERE conversation_id = ?`).all(conversationId);const depthBySummaryId=new Map;const unresolvedCondensedIds=new Set;for(const summary of summaries){if(summary.kind==="leaf"){depthBySummaryId.set(summary.summary_id,0);continue}unresolvedCondensedIds.add(summary.summary_id)}const edges=db.prepare(`SELECT summary_id, parent_summary_id
|
|
97
|
+
FROM summary_parents
|
|
98
|
+
WHERE summary_id IN (
|
|
99
|
+
SELECT summary_id FROM summaries
|
|
100
|
+
WHERE conversation_id = ? AND kind = 'condensed'
|
|
101
|
+
)`).all(conversationId);const parentsBySummaryId=new Map;for(const edge of edges){const existing=parentsBySummaryId.get(edge.summary_id)??[];existing.push(edge.parent_summary_id);parentsBySummaryId.set(edge.summary_id,existing)}while(unresolvedCondensedIds.size>0){let progressed=false;for(const summaryId of[...unresolvedCondensedIds]){const parentIds=parentsBySummaryId.get(summaryId)??[];if(parentIds.length===0){depthBySummaryId.set(summaryId,1);unresolvedCondensedIds.delete(summaryId);progressed=true;continue}let maxParentDepth=-1;let allParentsResolved=true;for(const parentId of parentIds){const parentDepth=depthBySummaryId.get(parentId);if(parentDepth==null){allParentsResolved=false;break}if(parentDepth>maxParentDepth){maxParentDepth=parentDepth}}if(!allParentsResolved){continue}depthBySummaryId.set(summaryId,maxParentDepth+1);unresolvedCondensedIds.delete(summaryId);progressed=true}if(!progressed){for(const summaryId of unresolvedCondensedIds){depthBySummaryId.set(summaryId,1)}unresolvedCondensedIds.clear()}}for(const summary of summaries){const depth=depthBySummaryId.get(summary.summary_id);if(depth==null){continue}updateDepthStmt.run(depth,summary.summary_id)}}}function backfillSummaryMetadata(db){const conversationRows=db.prepare(`SELECT DISTINCT conversation_id FROM summaries`).all();if(conversationRows.length===0){return}const updateMetadataStmt=db.prepare(`UPDATE summaries
|
|
102
|
+
SET earliest_at = ?, latest_at = ?, descendant_count = ?,
|
|
103
|
+
descendant_token_count = ?, source_message_token_count = ?
|
|
104
|
+
WHERE summary_id = ?`);for(const conversationRow of conversationRows){const conversationId=conversationRow.conversation_id;const summaries=db.prepare(`SELECT summary_id, conversation_id, kind, depth, token_count, created_at
|
|
105
|
+
FROM summaries
|
|
106
|
+
WHERE conversation_id = ?
|
|
107
|
+
ORDER BY depth ASC, created_at ASC`).all(conversationId);if(summaries.length===0){continue}const leafRanges=db.prepare(`SELECT
|
|
108
|
+
sm.summary_id,
|
|
109
|
+
MIN(m.created_at) AS earliest_at,
|
|
110
|
+
MAX(m.created_at) AS latest_at,
|
|
111
|
+
COALESCE(SUM(m.token_count), 0) AS source_message_token_count
|
|
112
|
+
FROM summary_messages sm
|
|
113
|
+
JOIN messages m ON m.message_id = sm.message_id
|
|
114
|
+
JOIN summaries s ON s.summary_id = sm.summary_id
|
|
115
|
+
WHERE s.conversation_id = ? AND s.kind = 'leaf'
|
|
116
|
+
GROUP BY sm.summary_id`).all(conversationId);const leafRangeBySummaryId=new Map(leafRanges.map(row=>[row.summary_id,{earliestAt:row.earliest_at,latestAt:row.latest_at,sourceMessageTokenCount:row.source_message_token_count}]));const edges=db.prepare(`SELECT summary_id, parent_summary_id
|
|
117
|
+
FROM summary_parents
|
|
118
|
+
WHERE summary_id IN (
|
|
119
|
+
SELECT summary_id FROM summaries WHERE conversation_id = ?
|
|
120
|
+
)`).all(conversationId);const parentsBySummaryId=new Map;for(const edge of edges){const existing=parentsBySummaryId.get(edge.summary_id)??[];existing.push(edge.parent_summary_id);parentsBySummaryId.set(edge.summary_id,existing)}const metadataBySummaryId=new Map;const tokenCountBySummaryId=new Map(summaries.map(summary=>[summary.summary_id,Math.max(0,Math.floor(summary.token_count??0))]));for(const summary of summaries){const fallbackDate=parseTimestamp(summary.created_at);if(summary.kind==="leaf"){const range=leafRangeBySummaryId.get(summary.summary_id);const earliestAt2=parseTimestamp(range?.earliestAt??summary.created_at)??fallbackDate;const latestAt2=parseTimestamp(range?.latestAt??summary.created_at)??fallbackDate;metadataBySummaryId.set(summary.summary_id,{earliestAt:earliestAt2,latestAt:latestAt2,descendantCount:0,descendantTokenCount:0,sourceMessageTokenCount:Math.max(0,Math.floor(range?.sourceMessageTokenCount??0))});continue}const parentIds=parentsBySummaryId.get(summary.summary_id)??[];if(parentIds.length===0){metadataBySummaryId.set(summary.summary_id,{earliestAt:fallbackDate,latestAt:fallbackDate,descendantCount:0,descendantTokenCount:0,sourceMessageTokenCount:0});continue}let earliestAt=null;let latestAt=null;let descendantCount=0;let descendantTokenCount=0;let sourceMessageTokenCount=0;for(const parentId of parentIds){const parentMetadata=metadataBySummaryId.get(parentId);if(!parentMetadata){continue}const parentEarliest=parentMetadata.earliestAt;if(parentEarliest&&(!earliestAt||parentEarliest<earliestAt)){earliestAt=parentEarliest}const parentLatest=parentMetadata.latestAt;if(parentLatest&&(!latestAt||parentLatest>latestAt)){latestAt=parentLatest}descendantCount+=Math.max(0,parentMetadata.descendantCount)+1;const parentTokenCount=tokenCountBySummaryId.get(parentId)??0;descendantTokenCount+=Math.max(0,parentTokenCount)+Math.max(0,parentMetadata.descendantTokenCount);sourceMessageTokenCount+=Math.max(0,parentMetadata.sourceMessageTokenCount)}metadataBySummaryId.set(summary.summary_id,{earliestAt:earliestAt??fallbackDate,latestAt:latestAt??fallbackDate,descendantCount:Math.max(0,descendantCount),descendantTokenCount:Math.max(0,descendantTokenCount),sourceMessageTokenCount:Math.max(0,sourceMessageTokenCount)})}for(const summary of summaries){const metadata=metadataBySummaryId.get(summary.summary_id);if(!metadata){continue}updateMetadataStmt.run(isoStringOrNull(metadata.earliestAt),isoStringOrNull(metadata.latestAt),Math.max(0,metadata.descendantCount),Math.max(0,metadata.descendantTokenCount),Math.max(0,metadata.sourceMessageTokenCount),summary.summary_id)}}}function backfillToolCallColumns(db){db.exec(`UPDATE message_parts
|
|
121
|
+
SET tool_call_id = COALESCE(
|
|
122
|
+
json_extract(metadata, '$.toolCallId'),
|
|
123
|
+
json_extract(metadata, '$.raw.id'),
|
|
124
|
+
json_extract(metadata, '$.raw.call_id'),
|
|
125
|
+
json_extract(metadata, '$.raw.toolCallId'),
|
|
126
|
+
json_extract(metadata, '$.raw.tool_call_id')
|
|
127
|
+
)
|
|
128
|
+
WHERE tool_call_id IS NULL
|
|
129
|
+
AND metadata IS NOT NULL
|
|
130
|
+
AND COALESCE(
|
|
131
|
+
json_extract(metadata, '$.toolCallId'),
|
|
132
|
+
json_extract(metadata, '$.raw.id'),
|
|
133
|
+
json_extract(metadata, '$.raw.call_id'),
|
|
134
|
+
json_extract(metadata, '$.raw.toolCallId'),
|
|
135
|
+
json_extract(metadata, '$.raw.tool_call_id')
|
|
136
|
+
) IS NOT NULL`);db.exec(`UPDATE message_parts
|
|
137
|
+
SET tool_name = COALESCE(
|
|
138
|
+
json_extract(metadata, '$.toolName'),
|
|
139
|
+
json_extract(metadata, '$.raw.name'),
|
|
140
|
+
json_extract(metadata, '$.raw.toolName'),
|
|
141
|
+
json_extract(metadata, '$.raw.tool_name')
|
|
142
|
+
)
|
|
143
|
+
WHERE tool_name IS NULL
|
|
144
|
+
AND metadata IS NOT NULL
|
|
145
|
+
AND COALESCE(
|
|
146
|
+
json_extract(metadata, '$.toolName'),
|
|
147
|
+
json_extract(metadata, '$.raw.name'),
|
|
148
|
+
json_extract(metadata, '$.raw.toolName'),
|
|
149
|
+
json_extract(metadata, '$.raw.tool_name')
|
|
150
|
+
) IS NOT NULL`);db.exec(`UPDATE message_parts
|
|
151
|
+
SET tool_input = COALESCE(
|
|
152
|
+
json_extract(metadata, '$.raw.input'),
|
|
153
|
+
json_extract(metadata, '$.raw.arguments'),
|
|
154
|
+
json_extract(metadata, '$.raw.toolInput')
|
|
155
|
+
)
|
|
156
|
+
WHERE tool_input IS NULL
|
|
157
|
+
AND metadata IS NOT NULL
|
|
158
|
+
AND COALESCE(
|
|
159
|
+
json_extract(metadata, '$.raw.input'),
|
|
160
|
+
json_extract(metadata, '$.raw.arguments'),
|
|
161
|
+
json_extract(metadata, '$.raw.toolInput')
|
|
162
|
+
) IS NOT NULL`)}function getExistingTableNames(db,names){if(names.length===0){return new Set}const placeholders=names.map(()=>"?").join(", ");const rows=db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (${placeholders})`).all(...names);return new Set(rows.map(row=>row.name).filter(name=>typeof name==="string"&&name.length>0))}function getFtsShadowTableNames(tableName){return[`${tableName}_data`,`${tableName}_idx`,`${tableName}_content`,`${tableName}_docsize`,`${tableName}_config`]}function quoteSqlIdentifier(identifier){if(!/^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)){throw new Error(`Invalid SQL identifier: ${identifier}`)}return`"${identifier.replaceAll(`"`,`""`)}"`}function shouldRecreateStandaloneFtsTable(db,spec){const shadowTables=getFtsShadowTableNames(spec.tableName);const existingTables=getExistingTableNames(db,[spec.tableName,...shadowTables]);if(!existingTables.has(spec.tableName)){return true}if(shadowTables.some(name=>!existingTables.has(name))){return true}try{const info=db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name = ?").get(spec.tableName);const sql=info?.sql??"";if(spec.staleSchemaPatterns?.some(pattern=>sql.includes(pattern))){return true}const columns=db.prepare(`PRAGMA table_info(${quoteSqlIdentifier(spec.tableName)})`).all();const columnNames=new Set(columns.map(col=>col.name).filter(name=>typeof name==="string"&&name.length>0));return spec.expectedColumns.some(column=>!columnNames.has(column))}catch{return true}}function ensureStandaloneFtsTable(db,spec){if(!shouldRecreateStandaloneFtsTable(db,spec)){return}db.exec(`DROP TABLE IF EXISTS ${quoteSqlIdentifier(spec.tableName)}`);for(const shadowTableName of getFtsShadowTableNames(spec.tableName)){db.exec(`DROP TABLE IF EXISTS ${quoteSqlIdentifier(shadowTableName)}`)}db.exec(spec.createSql);db.exec(spec.seedSql)}function runLcmMigrations(db,options){const log=options?.log;let transactionActive=false;try{db.exec(`PRAGMA busy_timeout = 30000`)}catch{}db.exec(`BEGIN EXCLUSIVE`);transactionActive=true;try{db.exec(`
|
|
163
|
+
CREATE TABLE IF NOT EXISTS conversations (
|
|
164
|
+
conversation_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
165
|
+
session_id TEXT NOT NULL,
|
|
166
|
+
session_key TEXT,
|
|
167
|
+
active INTEGER NOT NULL DEFAULT 1,
|
|
168
|
+
archived_at TEXT,
|
|
169
|
+
title TEXT,
|
|
170
|
+
bootstrapped_at TEXT,
|
|
171
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
172
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
173
|
+
);
|
|
174
|
+
|
|
175
|
+
CREATE TABLE IF NOT EXISTS messages (
|
|
176
|
+
message_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
177
|
+
conversation_id INTEGER NOT NULL REFERENCES conversations(conversation_id) ON DELETE CASCADE,
|
|
178
|
+
seq INTEGER NOT NULL,
|
|
179
|
+
role TEXT NOT NULL CHECK (role IN ('system', 'user', 'assistant', 'tool')),
|
|
180
|
+
content TEXT NOT NULL,
|
|
181
|
+
token_count INTEGER NOT NULL,
|
|
182
|
+
identity_hash TEXT,
|
|
183
|
+
transcript_entry_id TEXT,
|
|
184
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
185
|
+
UNIQUE (conversation_id, seq)
|
|
186
|
+
);
|
|
187
|
+
|
|
188
|
+
CREATE TABLE IF NOT EXISTS summaries (
|
|
189
|
+
summary_id TEXT PRIMARY KEY,
|
|
190
|
+
conversation_id INTEGER NOT NULL REFERENCES conversations(conversation_id) ON DELETE CASCADE,
|
|
191
|
+
kind TEXT NOT NULL CHECK (kind IN ('leaf', 'condensed')),
|
|
192
|
+
depth INTEGER NOT NULL DEFAULT 0,
|
|
193
|
+
content TEXT NOT NULL,
|
|
194
|
+
token_count INTEGER NOT NULL,
|
|
195
|
+
earliest_at TEXT,
|
|
196
|
+
latest_at TEXT,
|
|
197
|
+
descendant_count INTEGER NOT NULL DEFAULT 0,
|
|
198
|
+
descendant_token_count INTEGER NOT NULL DEFAULT 0,
|
|
199
|
+
source_message_token_count INTEGER NOT NULL DEFAULT 0,
|
|
200
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
201
|
+
file_ids TEXT NOT NULL DEFAULT '[]'
|
|
202
|
+
);
|
|
203
|
+
|
|
204
|
+
CREATE TABLE IF NOT EXISTS message_parts (
|
|
205
|
+
part_id TEXT PRIMARY KEY,
|
|
206
|
+
message_id INTEGER NOT NULL REFERENCES messages(message_id) ON DELETE CASCADE,
|
|
207
|
+
session_id TEXT NOT NULL,
|
|
208
|
+
part_type TEXT NOT NULL CHECK (part_type IN (
|
|
209
|
+
'text', 'reasoning', 'tool', 'patch', 'file',
|
|
210
|
+
'subtask', 'compaction', 'step_start', 'step_finish',
|
|
211
|
+
'snapshot', 'agent', 'retry'
|
|
212
|
+
)),
|
|
213
|
+
ordinal INTEGER NOT NULL,
|
|
214
|
+
text_content TEXT,
|
|
215
|
+
is_ignored INTEGER,
|
|
216
|
+
is_synthetic INTEGER,
|
|
217
|
+
tool_call_id TEXT,
|
|
218
|
+
tool_name TEXT,
|
|
219
|
+
tool_status TEXT,
|
|
220
|
+
tool_input TEXT,
|
|
221
|
+
tool_output TEXT,
|
|
222
|
+
tool_error TEXT,
|
|
223
|
+
tool_title TEXT,
|
|
224
|
+
patch_hash TEXT,
|
|
225
|
+
patch_files TEXT,
|
|
226
|
+
file_mime TEXT,
|
|
227
|
+
file_name TEXT,
|
|
228
|
+
file_url TEXT,
|
|
229
|
+
subtask_prompt TEXT,
|
|
230
|
+
subtask_desc TEXT,
|
|
231
|
+
subtask_agent TEXT,
|
|
232
|
+
step_reason TEXT,
|
|
233
|
+
step_cost REAL,
|
|
234
|
+
step_tokens_in INTEGER,
|
|
235
|
+
step_tokens_out INTEGER,
|
|
236
|
+
snapshot_hash TEXT,
|
|
237
|
+
compaction_auto INTEGER,
|
|
238
|
+
metadata TEXT,
|
|
239
|
+
UNIQUE (message_id, ordinal)
|
|
240
|
+
);
|
|
241
|
+
|
|
242
|
+
CREATE TABLE IF NOT EXISTS summary_messages (
|
|
243
|
+
summary_id TEXT NOT NULL REFERENCES summaries(summary_id) ON DELETE CASCADE,
|
|
244
|
+
message_id INTEGER NOT NULL REFERENCES messages(message_id) ON DELETE RESTRICT,
|
|
245
|
+
ordinal INTEGER NOT NULL,
|
|
246
|
+
PRIMARY KEY (summary_id, message_id)
|
|
247
|
+
);
|
|
248
|
+
|
|
249
|
+
CREATE TABLE IF NOT EXISTS summary_parents (
|
|
250
|
+
summary_id TEXT NOT NULL REFERENCES summaries(summary_id) ON DELETE CASCADE,
|
|
251
|
+
parent_summary_id TEXT NOT NULL REFERENCES summaries(summary_id) ON DELETE RESTRICT,
|
|
252
|
+
ordinal INTEGER NOT NULL,
|
|
253
|
+
PRIMARY KEY (summary_id, parent_summary_id)
|
|
254
|
+
);
|
|
255
|
+
|
|
256
|
+
CREATE TABLE IF NOT EXISTS context_items (
|
|
257
|
+
conversation_id INTEGER NOT NULL REFERENCES conversations(conversation_id) ON DELETE CASCADE,
|
|
258
|
+
ordinal INTEGER NOT NULL,
|
|
259
|
+
item_type TEXT NOT NULL CHECK (item_type IN ('message', 'summary')),
|
|
260
|
+
message_id INTEGER REFERENCES messages(message_id) ON DELETE RESTRICT,
|
|
261
|
+
summary_id TEXT REFERENCES summaries(summary_id) ON DELETE RESTRICT,
|
|
262
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
263
|
+
PRIMARY KEY (conversation_id, ordinal),
|
|
264
|
+
CHECK (
|
|
265
|
+
(item_type = 'message' AND message_id IS NOT NULL AND summary_id IS NULL) OR
|
|
266
|
+
(item_type = 'summary' AND summary_id IS NOT NULL AND message_id IS NULL)
|
|
267
|
+
)
|
|
268
|
+
);
|
|
269
|
+
|
|
270
|
+
CREATE TABLE IF NOT EXISTS large_files (
|
|
271
|
+
file_id TEXT PRIMARY KEY,
|
|
272
|
+
conversation_id INTEGER NOT NULL REFERENCES conversations(conversation_id) ON DELETE CASCADE,
|
|
273
|
+
file_name TEXT,
|
|
274
|
+
mime_type TEXT,
|
|
275
|
+
byte_size INTEGER,
|
|
276
|
+
storage_uri TEXT NOT NULL,
|
|
277
|
+
exploration_summary TEXT,
|
|
278
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
279
|
+
);
|
|
280
|
+
|
|
281
|
+
CREATE TABLE IF NOT EXISTS conversation_bootstrap_state (
|
|
282
|
+
conversation_id INTEGER PRIMARY KEY REFERENCES conversations(conversation_id) ON DELETE CASCADE,
|
|
283
|
+
session_file_path TEXT NOT NULL,
|
|
284
|
+
last_seen_size INTEGER NOT NULL,
|
|
285
|
+
last_seen_mtime_ms INTEGER NOT NULL,
|
|
286
|
+
last_processed_offset INTEGER NOT NULL,
|
|
287
|
+
last_processed_entry_hash TEXT,
|
|
288
|
+
session_header_id TEXT,
|
|
289
|
+
last_processed_entry_id TEXT,
|
|
290
|
+
fork_bounded INTEGER NOT NULL DEFAULT 0,
|
|
291
|
+
fork_source_message_count INTEGER NOT NULL DEFAULT 0,
|
|
292
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
293
|
+
);
|
|
294
|
+
|
|
295
|
+
CREATE TABLE IF NOT EXISTS conversation_compaction_telemetry (
|
|
296
|
+
conversation_id INTEGER PRIMARY KEY REFERENCES conversations(conversation_id) ON DELETE CASCADE,
|
|
297
|
+
last_observed_cache_read INTEGER,
|
|
298
|
+
last_observed_cache_write INTEGER,
|
|
299
|
+
last_observed_prompt_token_count INTEGER,
|
|
300
|
+
last_observed_cache_hit_at TEXT,
|
|
301
|
+
last_observed_cache_break_at TEXT,
|
|
302
|
+
cache_state TEXT NOT NULL DEFAULT 'unknown'
|
|
303
|
+
CHECK (cache_state IN ('hot', 'cold', 'unknown')),
|
|
304
|
+
consecutive_cold_observations INTEGER NOT NULL DEFAULT 0,
|
|
305
|
+
retention TEXT,
|
|
306
|
+
last_leaf_compaction_at TEXT,
|
|
307
|
+
turns_since_leaf_compaction INTEGER NOT NULL DEFAULT 0,
|
|
308
|
+
tokens_accumulated_since_leaf_compaction INTEGER NOT NULL DEFAULT 0,
|
|
309
|
+
last_activity_band TEXT NOT NULL DEFAULT 'low'
|
|
310
|
+
CHECK (last_activity_band IN ('low', 'medium', 'high')),
|
|
311
|
+
last_api_call_at TEXT,
|
|
312
|
+
last_cache_touch_at TEXT,
|
|
313
|
+
provider TEXT,
|
|
314
|
+
model TEXT,
|
|
315
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
316
|
+
);
|
|
317
|
+
|
|
318
|
+
CREATE TABLE IF NOT EXISTS conversation_compaction_maintenance (
|
|
319
|
+
conversation_id INTEGER PRIMARY KEY REFERENCES conversations(conversation_id) ON DELETE CASCADE,
|
|
320
|
+
pending INTEGER NOT NULL DEFAULT 0,
|
|
321
|
+
requested_at TEXT,
|
|
322
|
+
reason TEXT,
|
|
323
|
+
running INTEGER NOT NULL DEFAULT 0,
|
|
324
|
+
last_started_at TEXT,
|
|
325
|
+
last_finished_at TEXT,
|
|
326
|
+
last_failure_summary TEXT,
|
|
327
|
+
token_budget INTEGER,
|
|
328
|
+
current_token_count INTEGER,
|
|
329
|
+
projected_token_count INTEGER,
|
|
330
|
+
raw_tokens_outside_tail INTEGER,
|
|
331
|
+
context_threshold REAL,
|
|
332
|
+
context_threshold_source TEXT,
|
|
333
|
+
retry_attempts INTEGER NOT NULL DEFAULT 0,
|
|
334
|
+
next_attempt_after TEXT,
|
|
335
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
336
|
+
);
|
|
337
|
+
|
|
338
|
+
CREATE TABLE IF NOT EXISTS focus_briefs (
|
|
339
|
+
brief_id TEXT PRIMARY KEY,
|
|
340
|
+
conversation_id INTEGER NOT NULL REFERENCES conversations(conversation_id) ON DELETE CASCADE,
|
|
341
|
+
session_key TEXT,
|
|
342
|
+
prompt TEXT NOT NULL,
|
|
343
|
+
content TEXT NOT NULL,
|
|
344
|
+
status TEXT NOT NULL CHECK (status IN ('draft', 'active', 'superseded', 'failed', 'inactive')),
|
|
345
|
+
token_count INTEGER NOT NULL DEFAULT 0,
|
|
346
|
+
target_tokens INTEGER NOT NULL DEFAULT 0,
|
|
347
|
+
covered_latest_at TEXT,
|
|
348
|
+
covered_message_seq INTEGER,
|
|
349
|
+
source_context_hash TEXT NOT NULL DEFAULT '',
|
|
350
|
+
generator_run_id TEXT,
|
|
351
|
+
generator_session_key TEXT,
|
|
352
|
+
raw_result_json TEXT,
|
|
353
|
+
error TEXT,
|
|
354
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
355
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
356
|
+
superseded_at TEXT
|
|
357
|
+
);
|
|
358
|
+
|
|
359
|
+
CREATE TABLE IF NOT EXISTS focus_brief_sources (
|
|
360
|
+
brief_id TEXT NOT NULL REFERENCES focus_briefs(brief_id) ON DELETE CASCADE,
|
|
361
|
+
summary_id TEXT NOT NULL,
|
|
362
|
+
ordinal INTEGER,
|
|
363
|
+
role TEXT NOT NULL,
|
|
364
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
365
|
+
PRIMARY KEY (brief_id, summary_id, role)
|
|
366
|
+
);
|
|
367
|
+
|
|
368
|
+
CREATE TABLE IF NOT EXISTS lcm_migration_state (
|
|
369
|
+
step_name TEXT NOT NULL,
|
|
370
|
+
algorithm_version INTEGER NOT NULL,
|
|
371
|
+
completed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
372
|
+
PRIMARY KEY (step_name, algorithm_version)
|
|
373
|
+
);
|
|
374
|
+
|
|
375
|
+
-- Indexes
|
|
376
|
+
CREATE INDEX IF NOT EXISTS messages_conv_seq_idx ON messages (conversation_id, seq);
|
|
377
|
+
CREATE INDEX IF NOT EXISTS summaries_conv_created_idx ON summaries (conversation_id, created_at);
|
|
378
|
+
CREATE INDEX IF NOT EXISTS summary_messages_message_idx ON summary_messages (message_id);
|
|
379
|
+
CREATE INDEX IF NOT EXISTS summary_parents_parent_summary_idx ON summary_parents (parent_summary_id);
|
|
380
|
+
CREATE INDEX IF NOT EXISTS message_parts_message_idx ON message_parts (message_id);
|
|
381
|
+
CREATE INDEX IF NOT EXISTS message_parts_type_idx ON message_parts (part_type);
|
|
382
|
+
CREATE INDEX IF NOT EXISTS context_items_conv_idx ON context_items (conversation_id, ordinal);
|
|
383
|
+
CREATE INDEX IF NOT EXISTS large_files_conv_idx ON large_files (conversation_id, created_at);
|
|
384
|
+
CREATE INDEX IF NOT EXISTS bootstrap_state_path_idx
|
|
385
|
+
ON conversation_bootstrap_state (session_file_path, updated_at);
|
|
386
|
+
CREATE INDEX IF NOT EXISTS compaction_telemetry_state_idx
|
|
387
|
+
ON conversation_compaction_telemetry (cache_state, updated_at);
|
|
388
|
+
CREATE INDEX IF NOT EXISTS focus_briefs_conversation_status_idx
|
|
389
|
+
ON focus_briefs (conversation_id, status, created_at);
|
|
390
|
+
CREATE INDEX IF NOT EXISTS focus_brief_sources_summary_idx
|
|
391
|
+
ON focus_brief_sources (summary_id);
|
|
392
|
+
|
|
393
|
+
-- Speed up summary_messages lookups by message_id (PK is summary_id,message_id)
|
|
394
|
+
CREATE INDEX IF NOT EXISTS summary_messages_message_idx ON summary_messages (message_id);
|
|
395
|
+
`);const conversationColumns=db.prepare(`PRAGMA table_info(conversations)`).all();const hasBootstrappedAt=conversationColumns.some(col=>col.name==="bootstrapped_at");if(!hasBootstrappedAt){db.exec(`ALTER TABLE conversations ADD COLUMN bootstrapped_at TEXT`)}const hasSessionKey=conversationColumns.some(col=>col.name==="session_key");if(!hasSessionKey){db.exec(`ALTER TABLE conversations ADD COLUMN session_key TEXT`)}const hasActive=conversationColumns.some(col=>col.name==="active");if(!hasActive){db.exec(`ALTER TABLE conversations ADD COLUMN active INTEGER NOT NULL DEFAULT 1`)}const hasArchivedAt=conversationColumns.some(col=>col.name==="archived_at");if(!hasArchivedAt){db.exec(`ALTER TABLE conversations ADD COLUMN archived_at TEXT`)}db.exec(`UPDATE conversations SET active = 1 WHERE active IS NULL`);db.exec(`
|
|
396
|
+
CREATE UNIQUE INDEX IF NOT EXISTS conversations_active_session_key_idx
|
|
397
|
+
ON conversations (session_key)
|
|
398
|
+
WHERE session_key IS NOT NULL AND active = 1
|
|
399
|
+
`);db.exec(`
|
|
400
|
+
CREATE INDEX IF NOT EXISTS conversations_session_key_active_created_idx
|
|
401
|
+
ON conversations (session_key, active, created_at)
|
|
402
|
+
`);db.exec(`
|
|
403
|
+
CREATE INDEX IF NOT EXISTS conversations_session_id_active_created_idx
|
|
404
|
+
ON conversations (session_id, active, created_at)
|
|
405
|
+
`);db.exec(`DROP INDEX IF EXISTS conversations_session_key_idx`);runMigrationStep("ensureSummaryDepthColumn",log,()=>ensureSummaryDepthColumn(db));runMigrationStep("ensureSummaryMetadataColumns",log,()=>ensureSummaryMetadataColumns(db));runMigrationStep("ensureSummaryModelColumn",log,()=>ensureSummaryModelColumn(db));runMigrationStep("ensureMessageIdentityHashColumn",log,()=>ensureMessageIdentityHashColumn(db));runMigrationStep("ensureMessageLargeContentColumn",log,()=>ensureMessageLargeContentColumn(db));runMigrationStep("ensureConversationBootstrapStateForkColumns",log,()=>ensureConversationBootstrapStateForkColumns(db));runMigrationStep("ensureConversationBootstrapStateEpochColumns",log,()=>ensureConversationBootstrapStateEpochColumns(db));runMigrationStep("ensureMessagePartsTable",log,()=>ensureMessagePartsTable(db));runMigrationStep("backfillMessageIdentityHashes",log,()=>backfillMessageIdentityHashes(db,{managesOwnTransaction:false}));runVersionedBackfillStep(db,"repairOpenClawMetadataIdentityState",log,()=>repairOpenClawMetadataIdentityState(db));runMigrationStep("createMessagesIdentityHashIndex",log,()=>db.exec(`CREATE INDEX IF NOT EXISTS messages_conv_identity_hash_idx ON messages (conversation_id, identity_hash)`));runMigrationStep("ensureMessageTranscriptEntryIdColumn",log,()=>ensureMessageTranscriptEntryIdColumn(db));runMigrationStep("createMessagesTranscriptEntryIdIndex",log,()=>db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS messages_conv_entry_unique_idx
|
|
406
|
+
ON messages (conversation_id, transcript_entry_id)
|
|
407
|
+
WHERE transcript_entry_id IS NOT NULL`));runMigrationStep("ensureCompactionTelemetryColumns",log,()=>ensureCompactionTelemetryColumns(db));runMigrationStep("ensureCompactionMaintenanceColumns",log,()=>ensureCompactionMaintenanceColumns(db));runMigrationStep("ensureFocusBriefTables",log,()=>ensureFocusBriefTables(db));runVersionedBackfillStep(db,"backfillSummaryDepths",log,()=>backfillSummaryDepths(db));runMigrationStep("createSummariesDepthIndex",log,()=>db.exec(`CREATE INDEX IF NOT EXISTS summaries_conv_depth_kind_idx ON summaries (conversation_id, depth, kind)`));runVersionedBackfillStep(db,"backfillSummaryMetadata",log,()=>backfillSummaryMetadata(db));runVersionedBackfillStep(db,"backfillToolCallColumns",log,()=>backfillToolCallColumns(db));const detectedFeatures=options?.fts5Available===false?null:getLcmDbFeatures(db);const fts5Available=options?.fts5Available??detectedFeatures?.fts5Available??false;if(fts5Available){const trigramTokenizerAvailable=detectedFeatures?.trigramTokenizerAvailable??false;if(!trigramTokenizerAvailable){try{db.exec(`DROP TABLE IF EXISTS summaries_fts_cjk`)}catch{}}runMigrationStep("ensureMessagesFts",log,()=>{ensureStandaloneFtsTable(db,{tableName:"messages_fts",createSql:`
|
|
408
|
+
CREATE VIRTUAL TABLE messages_fts USING fts5(
|
|
409
|
+
content,
|
|
410
|
+
tokenize='porter unicode61'
|
|
411
|
+
)
|
|
412
|
+
`,seedSql:`
|
|
413
|
+
INSERT INTO messages_fts(rowid, content)
|
|
414
|
+
SELECT message_id, content FROM messages
|
|
415
|
+
`,expectedColumns:["content"],staleSchemaPatterns:["content_rowid"]})});runMigrationStep("ensureSummariesFts",log,()=>{ensureStandaloneFtsTable(db,{tableName:"summaries_fts",createSql:`
|
|
416
|
+
CREATE VIRTUAL TABLE summaries_fts USING fts5(
|
|
417
|
+
summary_id UNINDEXED,
|
|
418
|
+
content,
|
|
419
|
+
tokenize='porter unicode61'
|
|
420
|
+
)
|
|
421
|
+
`,seedSql:`
|
|
422
|
+
INSERT INTO summaries_fts(summary_id, content)
|
|
423
|
+
SELECT summary_id, content FROM summaries
|
|
424
|
+
`,expectedColumns:["summary_id","content"],staleSchemaPatterns:["content_rowid='summary_id'",'content_rowid="summary_id"']})});runMigrationStep("ensureSummariesFtsCjk",log,()=>{if(trigramTokenizerAvailable){ensureStandaloneFtsTable(db,{tableName:"summaries_fts_cjk",createSql:`
|
|
425
|
+
CREATE VIRTUAL TABLE summaries_fts_cjk USING fts5(
|
|
426
|
+
summary_id UNINDEXED,
|
|
427
|
+
content,
|
|
428
|
+
tokenize='trigram'
|
|
429
|
+
)
|
|
430
|
+
`,seedSql:`
|
|
431
|
+
INSERT INTO summaries_fts_cjk(summary_id, content)
|
|
432
|
+
SELECT summary_id, content FROM summaries
|
|
433
|
+
`,expectedColumns:["summary_id","content"]})}})}db.exec(`COMMIT`);transactionActive=false}catch(error){if(transactionActive){try{db.exec(`ROLLBACK`)}catch{}}throw error}}function isCjkCodePoint(cp){return cp>=19968&&cp<=40959||cp>=13312&&cp<=19903||cp>=131072&&cp<=173791||cp>=173824&&cp<=177983||cp>=177984&&cp<=178207||cp>=178208&&cp<=183983||cp>=183984&&cp<=191471||cp>=12288&&cp<=12351||cp>=12352&&cp<=12543||cp>=44032&&cp<=55215||cp>=65280&&cp<=65519}function estimateCodePointTokens(cp){if(isCjkCodePoint(cp)){return 1.5}if(cp>65535){return 2}return .25}function estimateTokens(text){let tokens=0;for(const char of text){const cp=char.codePointAt(0)??0;tokens+=estimateCodePointTokens(cp)}return Math.ceil(tokens)}function parseJson(value){if(typeof value!=="string"||!value.trim()){return void 0}try{return JSON.parse(value)}catch{return void 0}}function getPartMetadata(part){const decoded=parseJson(part.metadata);if(!decoded||typeof decoded!=="object"){return{}}const record=decoded;return{originalRole:typeof record.originalRole==="string"&&record.originalRole.length>0?record.originalRole:void 0,rawType:typeof record.rawType==="string"&&record.rawType.length>0?record.rawType:void 0,raw:record.raw,topLevelReasoningField:typeof record.topLevelReasoningField==="string"&&record.topLevelReasoningField.length>0?record.topLevelReasoningField:void 0,topLevelReasoningContent:typeof record.topLevelReasoningContent==="string"&&record.topLevelReasoningContent.length>0?record.topLevelReasoningContent:void 0,topLevelReasoningOnly:typeof record.topLevelReasoningOnly==="boolean"?record.topLevelReasoningOnly:void 0}}function parseStoredValue(value){if(typeof value!=="string"||value.length===0){return void 0}const parsed=parseJson(value);return parsed!==void 0?parsed:value}function reasoningBlockFromPart(part,rawType){const type=rawType==="thinking"?"thinking":"reasoning";if(typeof part.textContent==="string"&&part.textContent.length>0){return type==="thinking"?{type,thinking:part.textContent}:{type,text:part.textContent}}return{type}}function tryRestoreOpenAIReasoning(raw){if(raw.type!=="thinking")return null;const sig=raw.thinkingSignature;if(typeof sig!=="string"||!sig.startsWith("{"))return null;try{const parsed=JSON.parse(sig);if(parsed.type==="reasoning"&&typeof parsed.id==="string"){return parsed}}catch{}return null}function toolCallBlockFromPart(part,rawType){const type=rawType==="function_call"||rawType==="functionCall"||rawType==="tool_use"||rawType==="tool-use"||rawType==="toolUse"||rawType==="toolCall"?rawType:"toolCall";const input=parseStoredValue(part.toolInput);const block={type};if(type==="function_call"){if(typeof part.toolCallId==="string"&&part.toolCallId.length>0){block.call_id=part.toolCallId}if(typeof part.toolName==="string"&&part.toolName.length>0){block.name=part.toolName}if(input!==void 0){block.arguments=input}return block}block.id=typeof part.toolCallId==="string"&&part.toolCallId.length>0?part.toolCallId:`toolu_lcm_${part.partId??"unknown"}`;if(typeof part.toolName==="string"&&part.toolName.length>0){block.name=part.toolName}if(input!==void 0){if(type==="functionCall"||type==="toolCall"){block.arguments=input}else{block.input=input}}return block}function toolResultBlockFromPart(part,rawType,raw){if(raw&&typeof raw.text==="string"&&raw.output===void 0&&raw.content===void 0&&(part.toolOutput==null||part.toolOutput==="")&&(part.textContent==null||part.textContent===raw.text)){return{type:"text",text:raw.text}}const type=rawType==="function_call_output"||rawType==="toolResult"||rawType==="tool_result"?rawType:"tool_result";const output=parseStoredValue(part.toolOutput);const block={type};if(typeof part.toolName==="string"&&part.toolName.length>0){block.name=part.toolName}if(output!==void 0){block.output=output}else if(typeof part.textContent==="string"){block.output=part.textContent}else if(raw&&raw.output!==void 0){block.output=raw.output}else if(raw&&raw.content!==void 0){block.content=raw.content}else{block.output=""}if(raw&&typeof raw.is_error==="boolean"){block.is_error=raw.is_error}else if(raw&&typeof raw.isError==="boolean"){block.isError=raw.isError}if(type==="function_call_output"){if(typeof part.toolCallId==="string"&&part.toolCallId.length>0){block.call_id=part.toolCallId}return block}if(typeof part.toolCallId==="string"&&part.toolCallId.length>0){block.tool_use_id=part.toolCallId}return block}function blockFromPart(part){const metadata=getPartMetadata(part);if(metadata.raw&&typeof metadata.raw==="object"){const restored=tryRestoreOpenAIReasoning(metadata.raw);if(restored)return restored;const rawRecord=metadata.raw;const rawType=typeof rawRecord.type==="string"?rawRecord.type:metadata.rawType;if(rawType==="thinking"&&typeof rawRecord.thinkingSignature==="string"){const{thinkingSignature:_thinkingSignature,...cleaned}=rawRecord;return cleaned}const isToolBlock=rawType==="toolCall"||rawType==="tool_use"||rawType==="tool-use"||rawType==="toolUse"||rawType==="functionCall"||rawType==="function_call"||rawType==="function_call_output"||rawType==="toolResult"||rawType==="tool_result";if(!isToolBlock){return metadata.raw}const rawToolCallId=typeof rawRecord.id==="string"&&rawRecord.id.length>0?rawRecord.id:typeof rawRecord.call_id==="string"&&rawRecord.call_id.length>0?rawRecord.call_id:void 0;if(rawToolCallId){if(typeof part.toolCallId!=="string"||part.toolCallId.length===0){part.toolCallId=rawToolCallId}}if(typeof rawRecord.name==="string"&&rawRecord.name.length>0){if(typeof part.toolName!=="string"||part.toolName.length===0){part.toolName=rawRecord.name}}if(part.toolInput==null||part.toolInput===""){const rawArgs=rawRecord.arguments??rawRecord.input;if(rawArgs!==void 0){part.toolInput=typeof rawArgs==="string"?rawArgs:JSON.stringify(rawArgs)}}}if(part.partType==="reasoning"){return reasoningBlockFromPart(part,metadata.rawType)}if(part.partType==="tool"){if(metadata.originalRole==="toolResult"||metadata.rawType==="function_call_output"){return toolResultBlockFromPart(part,metadata.rawType,metadata.raw&&typeof metadata.raw==="object"?metadata.raw:void 0)}return toolCallBlockFromPart(part,metadata.rawType)}if(metadata.rawType==="function_call"||metadata.rawType==="functionCall"||metadata.rawType==="tool_use"||metadata.rawType==="tool-use"||metadata.rawType==="toolUse"||metadata.rawType==="toolCall"){return toolCallBlockFromPart(part,metadata.rawType)}if(metadata.rawType==="function_call_output"||metadata.rawType==="tool_result"||metadata.rawType==="toolResult"){return toolResultBlockFromPart(part,metadata.rawType,metadata.raw&&typeof metadata.raw==="object"?metadata.raw:void 0)}if(part.partType==="text"){return{type:"text",text:part.textContent??""}}if(typeof part.textContent==="string"&&part.textContent.length>0){return{type:"text",text:part.textContent}}const decodedFallback=parseJson(part.metadata);if(decodedFallback&&typeof decodedFallback==="object"){return{type:"text",text:JSON.stringify(decodedFallback)}}return{type:"text",text:""}}function toJson(value){const encoded=JSON.stringify(value);return typeof encoded==="string"?encoded:""}function safeString(value){return typeof value==="string"?value:void 0}function safeBoolean(value){return typeof value==="boolean"?value:void 0}function toRuntimeRoleForTokenEstimate(role){if(role==="tool"||role==="toolResult"){return"toolResult"}if(role==="user"||role==="system"){return"user"}return"assistant"}function estimateContentTokensForRole(params){const{role,content,fallbackContent}=params;if(typeof content==="string"){return estimateTokens(content)}if(Array.isArray(content)){if(content.length===0){return estimateTokens(fallbackContent)}if(role==="user"&&content.length===1&&isTextBlock(content[0])){return estimateTokens(content[0].text)}const serialized=JSON.stringify(content);return estimateTokens(typeof serialized==="string"?serialized:"")}if(content&&typeof content==="object"){if(role==="user"&&isTextBlock(content)){return estimateTokens(content.text)}const serialized=JSON.stringify([content]);return estimateTokens(typeof serialized==="string"?serialized:"")}return estimateTokens(fallbackContent)}function appendTextValue(value,out){if(typeof value==="string"){out.push(value);return}if(Array.isArray(value)){for(const entry of value){appendTextValue(entry,out)}return}if(!value||typeof value!=="object"){return}const record=value;appendTextValue(record.text,out);appendTextValue(record.value,out)}var STRUCTURED_TEXT_FIELD_KEYS=["text","transcript","transcription","message","summary"];var STRUCTURED_ARRAY_FIELD_KEYS=["segments","utterances","paragraphs","alternatives","words","items","results"];var STRUCTURED_NESTED_FIELD_KEYS=["content","output","result","payload","data","value"];var MAX_STRUCTURED_TEXT_DEPTH=6;var TOOL_CALL_RAW_TYPES=new Set(["tool_use","toolUse","tool-use","toolCall","tool_call","functionCall","function_call"]);var TOOL_RESULT_RAW_TYPES=new Set(["function_call_output","tool_result","toolResult","tool_use_result"]);var TOOL_RAW_TYPES=new Set([...TOOL_CALL_RAW_TYPES,...TOOL_RESULT_RAW_TYPES]);var REASONING_RAW_TYPES=new Set(["thinking","redacted_thinking","reasoning"]);var REPLAY_CRITICAL_RAW_TYPES=new Set([...TOOL_RAW_TYPES,...REASONING_RAW_TYPES]);function looksLikeJsonPayload(value){if(typeof value!=="string")return false;const trimmed=value.trim();if(!trimmed){return false}return trimmed.startsWith("{")&&trimmed.endsWith("}")||trimmed.startsWith("[")&&trimmed.endsWith("]")}function extractStructuredText(value,depth=0){if(value==null||depth>MAX_STRUCTURED_TEXT_DEPTH){return void 0}if(typeof value==="string"){if(looksLikeJsonPayload(value)){try{const parsed=JSON.parse(value.trim());const parsedText=extractStructuredText(parsed,depth+1);if(typeof parsedText==="string"&&parsedText.length>0){return parsedText}}catch{}}return value}if(Array.isArray(value)){const texts=[];for(const entry of value){const text=extractStructuredText(entry,depth+1);if(typeof text==="string"&&text.trim().length>0){texts.push(text)}}return texts.length>0?texts.join("\n"):void 0}if(typeof value!=="object"){return void 0}const record=value;if(typeof record.type==="string"&&REASONING_RAW_TYPES.has(record.type)){return void 0}if(typeof record.type==="string"&&TOOL_RAW_TYPES.has(record.type)){if(safeBoolean(record.toolOutputExternalized)){const externalizedText=extractStructuredText(record.output,depth+1)??extractStructuredText(record.content,depth+1)??extractStructuredText(record.result,depth+1);if(typeof externalizedText==="string"&&externalizedText.trim().length>0){return externalizedText}}return void 0}for(const key of STRUCTURED_TEXT_FIELD_KEYS){const candidate=record[key];if(typeof candidate==="string"&&candidate.trim().length>0){return candidate}}for(const key of STRUCTURED_ARRAY_FIELD_KEYS){const candidate=record[key];if(Array.isArray(candidate)){const texts=[];for(const entry of candidate){const text=extractStructuredText(entry,depth+1);if(typeof text==="string"&&text.trim().length>0){texts.push(text)}}if(texts.length>0){return texts.join("\n")}}}for(const key of STRUCTURED_NESTED_FIELD_KEYS){const nested=record[key];const nestedText=extractStructuredText(nested,depth+1);if(typeof nestedText==="string"&&nestedText.trim().length>0){return nestedText}}return void 0}function extractReasoningText(record){const chunks=[];appendTextValue(record.summary,chunks);if(chunks.length===0){return void 0}const normalized=chunks.map(chunk=>chunk.trim()).filter((chunk,idx,arr)=>chunk.length>0&&arr.indexOf(chunk)===idx);return normalized.length>0?normalized.join("\n"):void 0}function normalizeUnknownBlock(value){if(!value||typeof value!=="object"||Array.isArray(value)){return{type:"agent",metadata:{raw:value}}}const record=value;const rawType=safeString(record.type);return{type:rawType??"agent",text:safeString(record.text)??safeString(record.thinking)??(rawType==="reasoning"||rawType==="thinking"?extractReasoningText(record):void 0),metadata:{raw:record}}}function extractTopLevelReasoningContent(role,topLevel){if(role!=="assistant"){return null}const content=safeString(topLevel.reasoning_content);return content&&content.trim().length>0?{field:"reasoning_content",content}:null}function topLevelReasoningMetadata(reasoning,only=false){if(!reasoning){return{}}return{topLevelReasoningField:reasoning.field,topLevelReasoningContent:reasoning.content,topLevelReasoningOnly:only||void 0}}function toPartType(type){switch(type){case"text":return"text";case"thinking":case"redacted_thinking":case"reasoning":return"reasoning";case"tool_use":case"toolUse":case"tool-use":case"toolCall":case"functionCall":case"function_call":case"function_call_output":case"tool_result":case"toolResult":case"tool":return"tool";case"patch":return"patch";case"file":case"image":return"file";case"subtask":return"subtask";case"compaction":return"compaction";case"step_start":case"step-start":return"step_start";case"step_finish":case"step-finish":return"step_finish";case"snapshot":return"snapshot";case"retry":return"retry";case"agent":return"agent";default:return"agent"}}function extractMessageContent(content){const extracted=extractStructuredText(content);if(typeof extracted==="string"){return extracted}if(content==null){return""}if(Array.isArray(content)&&content.length===0){return""}if(Array.isArray(content)&&content.length>0&&content.every(item=>typeof item==="object"&&item!==null&&!Array.isArray(item)&&typeof item.type==="string"&&(TOOL_RAW_TYPES.has(item.type)||REASONING_RAW_TYPES.has(item.type)))){return""}const serialized=JSON.stringify(content);return typeof serialized==="string"?serialized:""}function isTextBlock(value){if(!value||typeof value!=="object"||Array.isArray(value)){return false}const record=value;return record.type==="text"&&typeof record.text==="string"}function toSyntheticMessagePartRecord(part,messageId){return{partId:`estimate-part-${part.ordinal}`,messageId,sessionId:part.sessionId,partType:part.partType,ordinal:part.ordinal,textContent:part.textContent??null,toolCallId:part.toolCallId??null,toolName:part.toolName??null,toolInput:part.toolInput??null,toolOutput:part.toolOutput??null,metadata:part.metadata??null}}function normalizeMessageContentForStorage(params){const{message,fallbackContent}=params;if(!("content"in message)){return fallbackContent}const role=toRuntimeRoleForTokenEstimate(message.role);const parts=buildMessageParts({sessionId:"storage-estimate",message,fallbackContent}).map(part=>toSyntheticMessagePartRecord(part,0));if(parts.length===0){if(role==="assistant"){return fallbackContent?[{type:"text",text:fallbackContent}]:[]}if(role==="toolResult"){return[{type:"text",text:fallbackContent}]}return fallbackContent}const blocks=parts.map(blockFromPart);if(role==="user"&&blocks.length===1&&isTextBlock(blocks[0])){return blocks[0].text}return blocks}function buildMessageParts(params){const{sessionId,message,fallbackContent}=params;const role=typeof message.role==="string"?message.role:"unknown";const topLevel=message;const topLevelToolCallId=safeString(topLevel.toolCallId)??safeString(topLevel.tool_call_id)??safeString(topLevel.toolUseId)??safeString(topLevel.tool_use_id)??safeString(topLevel.call_id)??safeString(topLevel.id);const topLevelToolName=safeString(topLevel.toolName)??safeString(topLevel.tool_name);const topLevelIsError=safeBoolean(topLevel.isError)??safeBoolean(topLevel.is_error);const topLevelReasoning=extractTopLevelReasoningContent(role,topLevel);const rawPayloadExternalized=safeBoolean(topLevel.rawPayloadExternalized);const externalizedFileId=safeString(topLevel.externalizedFileId);const externalizedFileIds=Array.isArray(topLevel.externalizedFileIds)?topLevel.externalizedFileIds.filter(fileId=>typeof fileId==="string"):void 0;const fileBlocksExternalized=safeBoolean(topLevel.fileBlocksExternalized);const originalByteSize=typeof topLevel.originalByteSize==="number"?topLevel.originalByteSize:void 0;const externalizationReason=safeString(topLevel.externalizationReason);if(!("content"in message)&&"command"in message&&"output"in message){return[{sessionId,partType:"text",ordinal:0,textContent:fallbackContent,metadata:toJson({originalRole:role,source:"bash-exec",command:safeString(message.command)})}]}if(!("content"in message)){return[{sessionId,partType:"agent",ordinal:0,textContent:fallbackContent||null,metadata:toJson({originalRole:role,source:"unknown-message-shape",raw:message})}]}if(typeof message.content==="string"){return[{sessionId,partType:"text",ordinal:0,textContent:message.content,metadata:toJson({originalRole:role,toolCallId:topLevelToolCallId,toolName:topLevelToolName,isError:topLevelIsError,...topLevelReasoningMetadata(topLevelReasoning),rawPayloadExternalized:rawPayloadExternalized||void 0,externalizedFileId,externalizedFileIds,fileBlocksExternalized:fileBlocksExternalized||void 0,originalByteSize,externalizationReason})}]}if(!Array.isArray(message.content)){return[{sessionId,partType:"agent",ordinal:0,textContent:fallbackContent||null,metadata:toJson({originalRole:role,source:"non-array-content",raw:message.content,...topLevelReasoningMetadata(topLevelReasoning)})}]}const parts=[];if(message.content.length===0&&topLevelReasoning){parts.push({sessionId,partType:"reasoning",ordinal:0,textContent:null,metadata:toJson({originalRole:role,rawType:topLevelReasoning.field,...topLevelReasoningMetadata(topLevelReasoning,true)})})}for(let ordinal=0;ordinal<message.content.length;ordinal++){const block=normalizeUnknownBlock(message.content[ordinal]);const metadataRecord=block.metadata.raw;const rawBlockType=safeString(metadataRecord?.rawType)??block.type;const partType=toPartType(rawBlockType);const rawBlock=metadataRecord&&rawBlockType!==block.type?{...metadataRecord,type:rawBlockType}:metadataRecord??message.content[ordinal];const toolCallId=safeString(metadataRecord?.toolCallId)??safeString(metadataRecord?.tool_call_id)??safeString(metadataRecord?.toolUseId)??safeString(metadataRecord?.tool_use_id)??safeString(metadataRecord?.call_id)??(partType==="tool"?safeString(metadataRecord?.id):void 0)??topLevelToolCallId;parts.push({sessionId,partType,ordinal,textContent:block.text??null,toolCallId,toolName:safeString(metadataRecord?.name)??safeString(metadataRecord?.toolName)??safeString(metadataRecord?.tool_name)??topLevelToolName,toolInput:metadataRecord?.input!==void 0?toJson(metadataRecord.input):metadataRecord?.arguments!==void 0?toJson(metadataRecord.arguments):metadataRecord?.toolInput!==void 0?toJson(metadataRecord.toolInput):safeString(metadataRecord?.tool_input)??null,toolOutput:metadataRecord?.output!==void 0?toJson(metadataRecord.output):metadataRecord?.toolOutput!==void 0?toJson(metadataRecord.toolOutput):safeString(metadataRecord?.tool_output)??null,metadata:toJson({originalRole:role,toolCallId:topLevelToolCallId,toolName:topLevelToolName,isError:topLevelIsError,...ordinal===0?topLevelReasoningMetadata(topLevelReasoning):{},externalizedFileId:safeString(metadataRecord?.externalizedFileId),originalByteSize:typeof metadataRecord?.originalByteSize==="number"?metadataRecord.originalByteSize:void 0,imageExternalized:safeBoolean(metadataRecord?.imageExternalized),toolOutputExternalized:safeBoolean(metadataRecord?.toolOutputExternalized),externalizationReason:safeString(metadataRecord?.externalizationReason),rawType:rawBlockType,raw:rawBlock})})}return parts}function toDbRole(role){if(role==="tool"||role==="toolResult"){return"tool"}if(role==="system"){return"system"}if(role==="user"){return"user"}if(role==="assistant"){return"assistant"}return"assistant"}function hasPersistableMessageRole(message){const role=message.role;return role==="user"||role==="assistant"||role==="system"||role==="tool"||role==="toolResult"}function filterPersistableMessages(messages){return messages.filter(hasPersistableMessageRole)}function toStoredMessage(message){const content="content"in message?extractMessageContent(message.content):"output"in message?`$ ${String(message.command??"")}
|
|
434
|
+
${String(message.output)}`:"";const runtimeRole=toRuntimeRoleForTokenEstimate(message.role);const normalizedContent="content"in message?normalizeMessageContentForStorage({message,fallbackContent:content}):content;const tokenCount="content"in message?estimateContentTokensForRole({role:runtimeRole,content:normalizedContent,fallbackContent:content}):estimateTokens(content);const topLevelReasoning=extractTopLevelReasoningContent(typeof message.role==="string"?message.role:"",message);return{role:toDbRole(message.role),content,tokenCount:tokenCount+(topLevelReasoning?estimateTokens(topLevelReasoning.content):0)}}import{createReadStream}from"node:fs";import{createInterface}from"node:readline";var TRANSCRIPT_ENTRY_META=Symbol.for("lossless-claw.transcriptEntryMeta");function attachTranscriptEntryMeta(message,meta){message[TRANSCRIPT_ENTRY_META]=meta;return message}function getTranscriptEntryMeta(message){const meta=message[TRANSCRIPT_ENTRY_META];return meta??null}function getTranscriptEntryId(message){return getTranscriptEntryMeta(message)?.entryId??null}function resolveTranscriptMessageCreatedAt(message){const raw=message;const value=raw.timestamp??raw.createdAt??raw.created_at;if(typeof value==="number"){const parsed=new Date(value);return Number.isFinite(parsed.getTime())?parsed:void 0}if(value instanceof Date){return Number.isFinite(value.getTime())?value:void 0}if(typeof value==="string"&&value.trim()){return value}return getTranscriptEntryMeta(message)?.timestamp??void 0}function normalizeEnvelopeString(value){if(typeof value!=="string"){return null}const trimmed=value.trim();return trimmed.length>0?trimmed:null}function extractEnvelopeMeta(entry){return{entryId:normalizeEnvelopeString(entry.id)??normalizeEnvelopeString(entry.uuid),parentId:normalizeEnvelopeString(entry.parentId)??normalizeEnvelopeString(entry.parentUuid),timestamp:normalizeEnvelopeString(entry.timestamp)}}async function readTranscriptHeader(sessionFile){const empty={sessionHeaderId:null,parentSession:null};try{const stream=createReadStream(sessionFile,{encoding:"utf8"});const lines=createInterface({input:stream,crlfDelay:Infinity});try{for await(const line of lines){const trimmed=line.trim();if(!trimmed){continue}try{const parsed=JSON.parse(trimmed);if(parsed.type!=="session"){return empty}return{sessionHeaderId:normalizeEnvelopeString(parsed.id),parentSession:normalizeEnvelopeString(parsed.parentSession)}}catch{return empty}}}finally{lines.close();stream.destroy()}}catch{return empty}return empty}function isBootstrapMessage(value){if(!value||typeof value!=="object"){return false}const msg=value;if(typeof msg.role!=="string"){return false}return"content"in msg||"command"in msg&&"output"in msg}function extractCanonicalBootstrapMessage(value){if(isBootstrapMessage(value)){return value}if(!value||typeof value!=="object"||Array.isArray(value)){return null}const entry=value;if("message"in entry){if(entry.type!==void 0&&entry.type!=="message"){return null}if(!isBootstrapMessage(entry.message)){return null}return attachTranscriptEntryMeta(entry.message,extractEnvelopeMeta(entry))}return null}function extractBootstrapMessageCandidate(value){return extractCanonicalBootstrapMessage(value)}function selectLeafPathRecords(records){if(records.length===0){return null}const byId=new Map;for(const record of records){if(!record.entryId){return null}byId.set(record.entryId,record)}const path=[];const visited=new Set;let current=records[records.length-1];while(current){const currentId=current.entryId;if(visited.has(currentId)){return null}visited.add(currentId);path.push(current);if(current.parentId===null){break}const parent=byId.get(current.parentId);if(!parent){return null}current=parent}path.reverse();return path}async function readLeafPathMessages(sessionFile){try{let sawNonWhitespace=false;let jsonArrayMode=false;let jsonArrayBuffer="";const records=[];const flattened=[];const stream=createReadStream(sessionFile,{encoding:"utf8"});const lines=createInterface({input:stream,crlfDelay:Infinity});for await(const line of lines){if(!sawNonWhitespace){const trimmed=line.trim();if(trimmed){sawNonWhitespace=true;if(trimmed.startsWith("[")){jsonArrayMode=true}}}if(jsonArrayMode){jsonArrayBuffer+=`${line}
|
|
435
|
+
`;continue}const item=line.trim();if(!item){continue}let parsed;try{parsed=JSON.parse(item)}catch{continue}if(parsed!==null&&typeof parsed==="object"&&!Array.isArray(parsed)&&parsed.type==="session"){continue}const candidate=extractBootstrapMessageCandidate(parsed);if(candidate){flattened.push(candidate)}if(parsed!==null&&typeof parsed==="object"&&!Array.isArray(parsed)){const meta=extractEnvelopeMeta(parsed);if(meta.entryId!==null||candidate){records.push({entryId:meta.entryId,parentId:meta.parentId,message:candidate})}}}if(jsonArrayMode){const trimmed=jsonArrayBuffer.trim();if(!trimmed){return[]}try{const parsed=JSON.parse(trimmed);if(!Array.isArray(parsed)){return[]}return parsed.filter(isBootstrapMessage)}catch{return[]}}const leafPath=selectLeafPathRecords(records);if(leafPath){return leafPath.map(record=>record.message).filter(message=>message!==null)}return flattened}catch{return[]}}import{createHash as createHash3}from"node:crypto";function createBootstrapEntryHash(message){if(!message){return null}const content=canonicalizeOpenClawInboundMetadataIdentityContent(message.role,message.content);return createHash3("sha256").update(JSON.stringify({role:message.role,content})).digest("hex")}import{randomUUID}from"node:crypto";import{AsyncLocalStorage}from"node:async_hooks";var mutexMap=new WeakMap;var heldLockContext=new AsyncLocalStorage;var nextSavepointId=0;function getOrCreateMutex(db){let state=mutexMap.get(db);if(!state){state={tail:Promise.resolve()};mutexMap.set(db,state)}return state}function getHeldLockDepth(db){return heldLockContext.getStore()?.get(db)??0}function nextSavepointName(){nextSavepointId+=1;return`lcm_txn_savepoint_${nextSavepointId}`}function acquireTransactionLock(db){const mutex=getOrCreateMutex(db);let releaseResolve;const releasePromise=new Promise(resolve3=>{releaseResolve=resolve3});const waitOn=mutex.tail;mutex.tail=releasePromise;return waitOn.then(()=>releaseResolve)}async function withDatabaseTransaction(db,beginStatement,operation){if(getHeldLockDepth(db)>0){const savepointName=nextSavepointName();db.exec(`SAVEPOINT ${savepointName}`);try{const result=await operation();db.exec(`RELEASE SAVEPOINT ${savepointName}`);return result}catch(error){db.exec(`ROLLBACK TO SAVEPOINT ${savepointName}`);db.exec(`RELEASE SAVEPOINT ${savepointName}`);throw error}}const release=await acquireTransactionLock(db);try{const heldLocks=new Map(heldLockContext.getStore()??[]);heldLocks.set(db,(heldLocks.get(db)??0)+1);return await heldLockContext.run(heldLocks,async()=>{db.exec(beginStatement);try{const result=await operation();db.exec("COMMIT");return result}catch(error){db.exec("ROLLBACK");throw error}})}finally{release()}}function appendConversationScopeConstraint(params){const normalizedConversationIds=[...new Set((params.conversationIds??[]).filter(value=>Number.isFinite(value)).map(value=>Math.trunc(value)))];if(normalizedConversationIds.length>0){if(normalizedConversationIds.length===1){params.where.push(`${params.columnExpr} = ?`);params.args.push(normalizedConversationIds[0]);return}params.where.push(`${params.columnExpr} IN (${normalizedConversationIds.map(()=>"?").join(", ")})`);params.args.push(...normalizedConversationIds);return}if(params.conversationId!=null){params.where.push(`${params.columnExpr} = ?`);params.args.push(params.conversationId)}}function sanitizeFts5Query(raw){const parts=[];const phraseRegex=/"([^"]+)"/g;let match;let lastIndex=0;while((match=phraseRegex.exec(raw))!==null){const before=raw.slice(lastIndex,match.index);for(const t of before.split(/\s+/).filter(Boolean)){parts.push(`"${t.replace(/"/g,"")}"`)}const phrase=match[1].replace(/"/g,"").trim();if(phrase){parts.push(`"${phrase}"`)}lastIndex=match.index+match[0].length}for(const t of raw.slice(lastIndex).split(/\s+/).filter(Boolean)){parts.push(`"${t.replace(/"/g,"")}"`)}return parts.length>0?parts.join(" "):'""'}var RAW_TERM_RE=/"([^"]+)"|(\S+)/g;var CJK_RE=/[\u2E80-\u9FFF\u3400-\u4DBF\uF900-\uFAFF\uAC00-\uD7AF\u3040-\u309F\u30A0-\u30FF]/;function containsCjk(text){return CJK_RE.test(text)}var EDGE_PUNCTUATION_RE=/^[`'"()[\]{}<>.,:;!?*_+=|\\/-]+|[`'"()[\]{}<>.,:;!?*_+=|\\/-]+$/g;function normalizeFallbackTerm(raw){if(typeof raw!=="string")return"";return raw.trim().replace(EDGE_PUNCTUATION_RE,"").toLowerCase()}function escapeLike(term){return term.replace(/([\\%_])/g,"\\$1")}function buildLikeSearchPlan(column,query){const terms=[];for(const match of query.matchAll(RAW_TERM_RE)){const raw=match[1]??match[2]??"";const normalized=normalizeFallbackTerm(raw);if(normalized.length>0&&!terms.includes(normalized)){terms.push(normalized)}}if(terms.length===0){const fallback=normalizeFallbackTerm(query);if(fallback.length>0){terms.push(fallback)}}return{terms,where:terms.map(()=>`LOWER(${column}) LIKE ? ESCAPE '\\'`),args:terms.map(term=>`%${escapeLike(term)}%`)}}function createFallbackSnippet(content,terms){const haystack=content.toLowerCase();let matchIndex=-1;let matchLength=0;for(const term of terms){const idx=haystack.indexOf(term);if(idx!==-1&&(matchIndex===-1||idx<matchIndex)){matchIndex=idx;matchLength=term.length}}if(matchIndex===-1){const head=content.trim();return head.length<=80?head:`${head.slice(0,77).trimEnd()}...`}const start=Math.max(0,matchIndex-24);const end=Math.min(content.length,matchIndex+Math.max(matchLength,1)+40);const prefix=start>0?"...":"";const suffix=end<content.length?"...":"";return`${prefix}${content.slice(start,end).trim()}${suffix}`}var AGE_DECAY_RATE=.001;function buildFtsOrderBy(sort,createdAtExpr){switch(sort??"recency"){case"relevance":return`rank ASC, ${createdAtExpr} DESC`;case"hybrid":return`(rank / (1 + ((julianday('now') - julianday(${createdAtExpr})) * 24 * ${AGE_DECAY_RATE}))) ASC, ${createdAtExpr} DESC`;default:return`${createdAtExpr} DESC`}}function toConversationRecord(row){return{conversationId:row.conversation_id,sessionId:row.session_id,sessionKey:row.session_key??null,active:row.active===1,archivedAt:parseUtcTimestampOrNull(row.archived_at),title:row.title,bootstrappedAt:parseUtcTimestampOrNull(row.bootstrapped_at),createdAt:parseUtcTimestamp(row.created_at),updatedAt:parseUtcTimestamp(row.updated_at)}}function formatMessageCreatedAt(value){if(value===void 0){return void 0}if(value instanceof Date){return Number.isFinite(value.getTime())?value.toISOString().slice(0,19).replace("T"," "):void 0}const trimmed=value.trim();if(!trimmed){return void 0}const parsed=parseUtcTimestampOrNull(trimmed);if(parsed&&Number.isFinite(parsed.getTime())){return parsed.toISOString().slice(0,19).replace("T"," ")}return void 0}function toMessageRecord(row){return{largeContent:row.large_content??null,messageId:row.message_id,conversationId:row.conversation_id,seq:row.seq,role:row.role,content:row.content,tokenCount:row.token_count,createdAt:parseUtcTimestamp(row.created_at)}}function toSearchResult(row){return{messageId:row.message_id,conversationId:row.conversation_id,role:row.role,snippet:row.snippet,createdAt:parseUtcTimestamp(row.created_at),rank:row.rank}}function toMessagePartRecord(row){return{partId:row.part_id,messageId:row.message_id,sessionId:row.session_id,partType:row.part_type,ordinal:row.ordinal,textContent:row.text_content,toolCallId:row.tool_call_id,toolName:row.tool_name,toolInput:row.tool_input,toolOutput:row.tool_output,metadata:row.metadata}}function normalizeMessageContentForFullTextIndex(content){if(typeof content!=="string")return null;const trimmed=content.trim();if(!trimmed){return null}const isExternalizedReference=trimmed.startsWith("[LCM File:")||trimmed.startsWith("[LCM Tool Output:");if(!isExternalizedReference){return content}const lines=trimmed.split(/\r?\n/).map(line=>line.trim()).filter(line=>line.length>0);if(lines.length===0){return null}const header=lines[0]??"";const summaryLines=[];let inSummary=false;for(let index=1;index<lines.length;index+=1){const line=lines[index];if(line==="Exploration Summary:"){inSummary=true;continue}if(line.startsWith("Use lcm_describe")||line.startsWith("Call lcm_describe")){continue}if(inSummary){summaryLines.push(line)}}const normalized=[header,...summaryLines].filter(line=>line.length>0).join("\n");return normalized||null}var ConversationStore=class{constructor(db,options){this.db=db;this.fts5Available=options?.fts5Available??true;this.replayFloodThresholdExternal=options?.replayFloodThresholdExternal??3;this.replayFloodThresholdInternal=options?.replayFloodThresholdInternal??32}db;fts5Available;replayFloodThresholdExternal;replayFloodThresholdInternal;async withTransaction(operation){return withDatabaseTransaction(this.db,"BEGIN IMMEDIATE",operation)}async createConversation(input){try{const result=this.db.prepare(`INSERT INTO conversations (session_id, session_key, active, archived_at, title)
|
|
436
|
+
VALUES (?, ?, ?, ?, ?)`).run(input.sessionId,input.sessionKey??null,input.active===false?0:1,input.archivedAt?.toISOString()??null,input.title??null);const row=this.db.prepare(`SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
|
|
437
|
+
FROM conversations WHERE conversation_id = ?`).get(Number(result.lastInsertRowid));return toConversationRecord(row)}catch(err){if(err instanceof Error&&/UNIQUE constraint failed|SQLITE_CONSTRAINT_UNIQUE/i.test(err.message)){if(input.sessionKey){const existing2=await this.getConversationBySessionKey(input.sessionKey);if(existing2)return existing2}const existing=await this.getActiveConversationBySessionId(input.sessionId);if(existing)return existing}throw err}}async getConversation(conversationId){const row=this.db.prepare(`SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
|
|
438
|
+
FROM conversations WHERE conversation_id = ?`).get(conversationId);return row?toConversationRecord(row):null}async getConversationBySessionId(sessionId){const row=this.db.prepare(`SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
|
|
439
|
+
FROM conversations
|
|
440
|
+
WHERE session_id = ?
|
|
441
|
+
ORDER BY active DESC, created_at DESC
|
|
442
|
+
LIMIT 1`).get(sessionId);return row?toConversationRecord(row):null}async getActiveConversationBySessionId(sessionId){const row=this.db.prepare(`SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
|
|
443
|
+
FROM conversations
|
|
444
|
+
WHERE session_id = ?
|
|
445
|
+
AND active = 1
|
|
446
|
+
ORDER BY created_at DESC
|
|
447
|
+
LIMIT 1`).get(sessionId);return row?toConversationRecord(row):null}async getConversationBySessionKey(sessionKey){const row=this.db.prepare(`SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
|
|
448
|
+
FROM conversations
|
|
449
|
+
WHERE session_key = ?
|
|
450
|
+
AND active = 1
|
|
451
|
+
ORDER BY created_at DESC
|
|
452
|
+
LIMIT 1`).get(sessionKey);return row?toConversationRecord(row):null}async getConversationFamilyIds(input){const baseConversation=input.conversationId!=null?await this.getConversation(input.conversationId):await this.getConversationForSession({sessionId:input.sessionId,sessionKey:input.sessionKey});if(!baseConversation){return[]}const normalizedSessionKey=baseConversation.sessionKey?.trim();if(normalizedSessionKey){const rows2=this.db.prepare(`SELECT conversation_id
|
|
453
|
+
FROM conversations
|
|
454
|
+
WHERE session_key = ?
|
|
455
|
+
ORDER BY active DESC, created_at DESC, conversation_id DESC`).all(normalizedSessionKey);return rows2.map(row=>row.conversation_id)}const rows=this.db.prepare(`SELECT conversation_id
|
|
456
|
+
FROM conversations
|
|
457
|
+
WHERE session_id = ?
|
|
458
|
+
ORDER BY active DESC, created_at DESC, conversation_id DESC`).all(baseConversation.sessionId);return rows.map(row=>row.conversation_id)}async getConversationForSession(input){const normalizedSessionKey=input.sessionKey?.trim();if(normalizedSessionKey){const byKey=await this.getConversationBySessionKey(normalizedSessionKey);if(byKey){return byKey}}const normalizedSessionId=input.sessionId?.trim();if(!normalizedSessionId){return null}return this.getActiveConversationBySessionId(normalizedSessionId)}async listActiveConversations(limit){const normalizedLimit=typeof limit==="number"&&Number.isFinite(limit)&&limit>0?Math.floor(limit):1e3;const rows=this.db.prepare(`SELECT conversation_id, session_id, session_key, active, archived_at, title, bootstrapped_at, created_at, updated_at
|
|
459
|
+
FROM conversations
|
|
460
|
+
WHERE active = 1
|
|
461
|
+
ORDER BY updated_at DESC, conversation_id DESC
|
|
462
|
+
LIMIT ?`).all(normalizedLimit);return rows.map(toConversationRecord)}async getOrCreateConversation(sessionId,titleOrOpts){const opts=typeof titleOrOpts==="string"?{title:titleOrOpts}:titleOrOpts??{};const normalizedSessionKey=opts.sessionKey?.trim();if(normalizedSessionKey){const byKey=await this.getConversationBySessionKey(normalizedSessionKey);if(byKey){if(byKey.sessionId!==sessionId){this.db.prepare(`UPDATE conversations SET session_id = ?, updated_at = datetime('now') WHERE conversation_id = ?`).run(sessionId,byKey.conversationId);byKey.sessionId=sessionId}return byKey}}const existing=await this.getActiveConversationBySessionId(sessionId);if(existing){if(!normalizedSessionKey){return existing}if(existing.active&&!existing.sessionKey){this.db.prepare(`UPDATE conversations SET session_key = ?, updated_at = datetime('now') WHERE conversation_id = ?`).run(normalizedSessionKey,existing.conversationId);existing.sessionKey=normalizedSessionKey;return existing}if(existing.active&&existing.sessionKey===normalizedSessionKey){return existing}}return this.createConversation({sessionId,title:opts.title,sessionKey:normalizedSessionKey})}async markConversationBootstrapped(conversationId){this.db.prepare(`UPDATE conversations
|
|
463
|
+
SET bootstrapped_at = COALESCE(bootstrapped_at, datetime('now')),
|
|
464
|
+
updated_at = datetime('now')
|
|
465
|
+
WHERE conversation_id = ?`).run(conversationId)}async archiveConversation(conversationId){this.db.prepare(`UPDATE conversations
|
|
466
|
+
SET active = 0,
|
|
467
|
+
archived_at = COALESCE(archived_at, datetime('now')),
|
|
468
|
+
updated_at = datetime('now')
|
|
469
|
+
WHERE conversation_id = ?`).run(conversationId)}async rebindConversationSession(conversationId,sessionId,sessionKey){const normalizedSessionId=sessionId.trim();const normalizedSessionKey=sessionKey?.trim()||null;if(!normalizedSessionId){return this.getConversation(conversationId)}this.db.prepare(`UPDATE conversations
|
|
470
|
+
SET session_id = ?,
|
|
471
|
+
session_key = COALESCE(?, session_key),
|
|
472
|
+
active = 1,
|
|
473
|
+
archived_at = NULL,
|
|
474
|
+
updated_at = datetime('now')
|
|
475
|
+
WHERE conversation_id = ?`).run(normalizedSessionId,normalizedSessionKey,conversationId);return this.getConversation(conversationId)}async createMessage(input){const prepared=this.prepareMessageInsert(input);if(!prepared.skipReplayTimestampFloodGuard){this.assertNoReplayTimestampFlood([prepared])}const result=this.db.prepare(`INSERT INTO messages (conversation_id, seq, role, content, token_count, identity_hash, transcript_entry_id, created_at)
|
|
476
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(prepared.conversationId,prepared.seq,prepared.role,prepared.content,prepared.tokenCount,prepared.identityHash,prepared.transcriptEntryId??null,prepared.createdAt);const messageId=Number(result.lastInsertRowid);this.indexMessageForFullText(messageId,input.content);const row=this.db.prepare(`SELECT message_id, conversation_id, seq, role, content, token_count, created_at, large_content
|
|
477
|
+
FROM messages WHERE message_id = ?`).get(messageId);return toMessageRecord(row)}async createMessagesBulk(inputs){if(inputs.length===0){return[]}const createdAt=this.currentSqliteTimestamp();const preparedInputs=inputs.map(input=>this.prepareMessageInsert(input,createdAt));this.assertNoReplayTimestampFlood(preparedInputs.filter(input=>!input.skipReplayTimestampFloodGuard));const insertStmt=this.db.prepare(`INSERT INTO messages (conversation_id, seq, role, content, token_count, identity_hash, transcript_entry_id, created_at)
|
|
478
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);const selectStmt=this.db.prepare(`SELECT message_id, conversation_id, seq, role, content, token_count, created_at, large_content
|
|
479
|
+
FROM messages WHERE message_id = ?`);const records=[];for(const input of preparedInputs){const result=insertStmt.run(input.conversationId,input.seq,input.role,input.content,input.tokenCount,input.identityHash,input.transcriptEntryId??null,input.createdAt);const messageId=Number(result.lastInsertRowid);this.indexMessageForFullText(messageId,input.content);const row=selectStmt.get(messageId);records.push(toMessageRecord(row))}return records}async getMessages(conversationId,opts){const afterSeq=opts?.afterSeq??-1;const limit=opts?.limit;if(limit!=null){const rows2=this.db.prepare(`SELECT message_id, conversation_id, seq, role, content, token_count, created_at, large_content
|
|
480
|
+
FROM messages
|
|
481
|
+
WHERE conversation_id = ? AND seq > ?
|
|
482
|
+
ORDER BY seq
|
|
483
|
+
LIMIT ?`).all(conversationId,afterSeq,limit);return rows2.map(toMessageRecord)}const rows=this.db.prepare(`SELECT message_id, conversation_id, seq, role, content, token_count, created_at, large_content
|
|
484
|
+
FROM messages
|
|
485
|
+
WHERE conversation_id = ? AND seq > ?
|
|
486
|
+
ORDER BY seq`).all(conversationId,afterSeq);return rows.map(toMessageRecord)}async getLastMessages(conversationId,count){if(count<=0){return[]}const rows=this.db.prepare(`SELECT message_id, conversation_id, seq, role, content, token_count, created_at, large_content
|
|
487
|
+
FROM messages
|
|
488
|
+
WHERE conversation_id = ?
|
|
489
|
+
ORDER BY seq DESC
|
|
490
|
+
LIMIT ?`).all(conversationId,Math.floor(count));return rows.reverse().map(toMessageRecord)}async getLastMessage(conversationId){const row=this.db.prepare(`SELECT message_id, conversation_id, seq, role, content, token_count, created_at, large_content
|
|
491
|
+
FROM messages
|
|
492
|
+
WHERE conversation_id = ?
|
|
493
|
+
ORDER BY seq DESC
|
|
494
|
+
LIMIT 1`).get(conversationId);return row?toMessageRecord(row):null}async getLastMessageIdentityHash(conversationId){const row=this.db.prepare(`SELECT identity_hash FROM messages
|
|
495
|
+
WHERE conversation_id = ?
|
|
496
|
+
ORDER BY seq DESC
|
|
497
|
+
LIMIT 1`).get(conversationId);return row?.identity_hash??null}async getRecentMessageIdentityHashes(conversationId,limit){const rows=this.db.prepare(`SELECT identity_hash FROM messages
|
|
498
|
+
WHERE conversation_id = ?
|
|
499
|
+
ORDER BY seq DESC
|
|
500
|
+
LIMIT ?`).all(conversationId,limit);return rows.map(r=>r.identity_hash).filter(h=>h!==null).reverse()}async hasMessage(conversationId,role,content){const identityHash=buildMessageIdentityHash(role,content);const row=this.db.prepare(`SELECT 1 AS count
|
|
501
|
+
FROM messages
|
|
502
|
+
WHERE conversation_id = ? AND identity_hash = ? AND role = ? AND content = ?
|
|
503
|
+
LIMIT 1`).get(conversationId,identityHash,role,content);return row?.count===1}async hasMessageByTranscriptEntryId(conversationId,transcriptEntryId){const row=this.db.prepare(`SELECT 1 AS count
|
|
504
|
+
FROM messages
|
|
505
|
+
WHERE conversation_id = ? AND transcript_entry_id = ?
|
|
506
|
+
LIMIT 1`).get(conversationId,transcriptEntryId);return row?.count===1}async adoptTranscriptEntryId(conversationId,role,content,transcriptEntryId){const identityHash=buildMessageIdentityHash(role,content);const result=this.db.prepare(`UPDATE messages
|
|
507
|
+
SET transcript_entry_id = ?
|
|
508
|
+
WHERE message_id = (
|
|
509
|
+
SELECT message_id
|
|
510
|
+
FROM messages
|
|
511
|
+
WHERE conversation_id = ?
|
|
512
|
+
AND transcript_entry_id IS NULL
|
|
513
|
+
AND identity_hash = ?
|
|
514
|
+
AND role = ?
|
|
515
|
+
AND content = ?
|
|
516
|
+
ORDER BY seq
|
|
517
|
+
LIMIT 1
|
|
518
|
+
)`).run(transcriptEntryId,conversationId,identityHash,role,content);return result.changes>0}async listTranscriptEntryIdsByIdentity(conversationId,role,content){const identityHash=buildMessageIdentityHash(role,content);const rows=this.db.prepare(`SELECT message_id, transcript_entry_id
|
|
519
|
+
FROM messages
|
|
520
|
+
WHERE conversation_id = ?
|
|
521
|
+
AND transcript_entry_id IS NOT NULL
|
|
522
|
+
AND identity_hash = ?
|
|
523
|
+
AND role = ?
|
|
524
|
+
AND content = ?
|
|
525
|
+
ORDER BY seq`).all(conversationId,identityHash,role,content);return rows.map(row=>({messageId:row.message_id,transcriptEntryId:row.transcript_entry_id}))}async restampTranscriptEntryId(messageId,transcriptEntryId){try{const result=this.db.prepare(`UPDATE messages SET transcript_entry_id = ? WHERE message_id = ?`).run(transcriptEntryId,messageId);return result.changes>0}catch(error){if(error instanceof Error&&error.message.includes("UNIQUE constraint failed")){return false}throw error}}async filterExistingTranscriptEntryIds(conversationId,entryIds){const existing=new Set;const chunkSize=400;for(let start=0;start<entryIds.length;start+=chunkSize){const chunk=entryIds.slice(start,start+chunkSize);const placeholders=chunk.map(()=>"?").join(", ");const rows=this.db.prepare(`SELECT transcript_entry_id
|
|
526
|
+
FROM messages
|
|
527
|
+
WHERE conversation_id = ? AND transcript_entry_id IN (${placeholders})`).all(conversationId,...chunk);for(const row of rows){existing.add(row.transcript_entry_id)}}return existing}async countMessagesByIdentity(conversationId,role,content){const identityHash=buildMessageIdentityHash(role,content);const row=this.db.prepare(`SELECT COUNT(*) AS count
|
|
528
|
+
FROM messages
|
|
529
|
+
WHERE conversation_id = ? AND identity_hash = ? AND role = ? AND content = ?`).get(conversationId,identityHash,role,content);return row?.count??0}async getNewestTranscriptEntryId(conversationId){const row=this.db.prepare(`SELECT transcript_entry_id AS id
|
|
530
|
+
FROM messages
|
|
531
|
+
WHERE conversation_id = ? AND transcript_entry_id IS NOT NULL
|
|
532
|
+
ORDER BY seq DESC
|
|
533
|
+
LIMIT 1`).get(conversationId);return row?.id??null}async hasRecentUnstampedMessageByIdentity(conversationId,role,content,tailWindow){const identityHash=buildMessageIdentityHash(role,content);const row=this.db.prepare(`SELECT 1 AS found
|
|
534
|
+
FROM (
|
|
535
|
+
SELECT message_id, transcript_entry_id, identity_hash, role, content
|
|
536
|
+
FROM messages
|
|
537
|
+
WHERE conversation_id = ?
|
|
538
|
+
ORDER BY seq DESC
|
|
539
|
+
LIMIT ?
|
|
540
|
+
)
|
|
541
|
+
WHERE transcript_entry_id IS NULL
|
|
542
|
+
AND identity_hash = ?
|
|
543
|
+
AND role = ?
|
|
544
|
+
AND content = ?
|
|
545
|
+
LIMIT 1`).get(conversationId,Math.max(1,Math.floor(tailWindow)),identityHash,role,content);return row?.found===1}async hasPreviousReasonedMessageByIdentity(conversationId,role,content){const identityHash=buildMessageIdentityHash(role,content);const row=this.db.prepare(`SELECT 1 AS found
|
|
546
|
+
FROM (
|
|
547
|
+
SELECT message_id, identity_hash, role, content
|
|
548
|
+
FROM messages
|
|
549
|
+
WHERE conversation_id = ?
|
|
550
|
+
ORDER BY seq DESC
|
|
551
|
+
LIMIT 1
|
|
552
|
+
) AS newest
|
|
553
|
+
WHERE newest.identity_hash = ?
|
|
554
|
+
AND newest.role = ?
|
|
555
|
+
AND newest.content = ?
|
|
556
|
+
AND EXISTS (
|
|
557
|
+
SELECT 1
|
|
558
|
+
FROM message_parts AS part
|
|
559
|
+
WHERE part.message_id = newest.message_id
|
|
560
|
+
AND (
|
|
561
|
+
part.part_type = 'reasoning'
|
|
562
|
+
OR (
|
|
563
|
+
part.metadata IS NOT NULL
|
|
564
|
+
AND json_valid(part.metadata)
|
|
565
|
+
AND json_extract(part.metadata, '$.topLevelReasoningField') = 'reasoning_content'
|
|
566
|
+
AND json_type(part.metadata, '$.topLevelReasoningContent') = 'text'
|
|
567
|
+
AND length(json_extract(part.metadata, '$.topLevelReasoningContent')) > 0
|
|
568
|
+
)
|
|
569
|
+
)
|
|
570
|
+
)
|
|
571
|
+
LIMIT 1`).get(conversationId,identityHash,role,content);return row?.found===1}async countMessagesByIdentityHash(conversationId,role,identityHash){const row=this.db.prepare(`SELECT COUNT(*) AS count
|
|
572
|
+
FROM messages
|
|
573
|
+
WHERE conversation_id = ? AND identity_hash = ? AND role = ?`).get(conversationId,identityHash,role);return row?.count??0}async countMessagesByIdentityBeforeTimestamp(params){return this.countMessagesByIdentityBeforeTimestampSync(params)}countMessagesByIdentityBeforeTimestampSync(params){const identityHash=buildMessageIdentityHash(params.role,params.content);const row=this.db.prepare(`SELECT COUNT(*) AS count
|
|
574
|
+
FROM messages
|
|
575
|
+
WHERE conversation_id = ?
|
|
576
|
+
AND identity_hash = ?
|
|
577
|
+
AND role = ?
|
|
578
|
+
AND content = ?
|
|
579
|
+
AND created_at < ?`).get(params.conversationId,identityHash,params.role,params.content,params.beforeCreatedAt);return row?.count??0}async getMessageById(messageId){const row=this.db.prepare(`SELECT message_id, conversation_id, seq, role, content, token_count, created_at, large_content
|
|
580
|
+
FROM messages WHERE message_id = ?`).get(messageId);return row?toMessageRecord(row):null}async getMessageByLargeContent(fileId){const row=this.db.prepare(`SELECT message_id, conversation_id, seq, role, content, token_count, created_at, large_content
|
|
581
|
+
FROM messages
|
|
582
|
+
WHERE large_content = ?
|
|
583
|
+
ORDER BY seq DESC
|
|
584
|
+
LIMIT 1`).get(fileId);return row?toMessageRecord(row):null}async createMessageParts(messageId,parts){if(parts.length===0){return}const stmt=this.db.prepare(`INSERT INTO message_parts (
|
|
585
|
+
part_id,
|
|
586
|
+
message_id,
|
|
587
|
+
session_id,
|
|
588
|
+
part_type,
|
|
589
|
+
ordinal,
|
|
590
|
+
text_content,
|
|
591
|
+
tool_call_id,
|
|
592
|
+
tool_name,
|
|
593
|
+
tool_input,
|
|
594
|
+
tool_output,
|
|
595
|
+
metadata
|
|
596
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);for(const part of parts){stmt.run(randomUUID(),messageId,part.sessionId,part.partType,part.ordinal,part.textContent??null,part.toolCallId??null,part.toolName??null,part.toolInput??null,part.toolOutput??null,part.metadata??null)}}async getMessageParts(messageId){const rows=this.db.prepare(`SELECT
|
|
597
|
+
part_id,
|
|
598
|
+
message_id,
|
|
599
|
+
session_id,
|
|
600
|
+
part_type,
|
|
601
|
+
ordinal,
|
|
602
|
+
text_content,
|
|
603
|
+
tool_call_id,
|
|
604
|
+
tool_name,
|
|
605
|
+
tool_input,
|
|
606
|
+
tool_output,
|
|
607
|
+
metadata
|
|
608
|
+
FROM message_parts
|
|
609
|
+
WHERE message_id = ?
|
|
610
|
+
ORDER BY ordinal`).all(messageId);return rows.map(toMessagePartRecord)}async getMessageCount(conversationId){const row=this.db.prepare(`SELECT COUNT(*) AS count FROM messages WHERE conversation_id = ?`).get(conversationId);return row?.count??0}async getMaxSeq(conversationId){const row=this.db.prepare(`SELECT COALESCE(MAX(seq), 0) AS max_seq
|
|
611
|
+
FROM messages WHERE conversation_id = ?`).get(conversationId);return row?.max_seq??0}async deleteMessages(messageIds){if(messageIds.length===0){return 0}let deleted=0;for(const messageId of messageIds){const refRow=this.db.prepare(`SELECT 1 AS found FROM summary_messages WHERE message_id = ? LIMIT 1`).get(messageId);if(refRow){continue}this.db.prepare(`DELETE FROM context_items WHERE item_type = 'message' AND message_id = ?`).run(messageId);this.deleteMessageFromFullText(messageId);this.db.prepare(`DELETE FROM messages WHERE message_id = ?`).run(messageId);deleted+=1}return deleted}async searchMessages(input){const limit=input.limit??50;if(input.mode==="full_text"){if(containsCjk(input.query)){return this.searchLike(input.query,limit,input.conversationId,input.conversationIds,input.since,input.before)}if(this.fts5Available){try{return this.searchFullText(input.query,limit,input.conversationId,input.conversationIds,input.since,input.before,input.sort)}catch{return this.searchLike(input.query,limit,input.conversationId,input.conversationIds,input.since,input.before)}}return this.searchLike(input.query,limit,input.conversationId,input.conversationIds,input.since,input.before)}return this.searchRegex(input.query,limit,input.conversationId,input.conversationIds,input.since,input.before)}indexMessageForFullText(messageId,content){if(!this.fts5Available){return}const normalizedContent=normalizeMessageContentForFullTextIndex(content);if(!normalizedContent){return}try{this.db.prepare(`INSERT INTO messages_fts(rowid, content) VALUES (?, ?)`).run(messageId,normalizedContent)}catch{}}currentSqliteTimestamp(){const row=this.db.prepare(`SELECT datetime('now') AS created_at`).get();return row.created_at}prepareMessageInsert(input,createdAt=this.currentSqliteTimestamp()){return{...input,createdAt:formatMessageCreatedAt(input.createdAt)??createdAt,identityHash:input.identityHash??buildMessageIdentityHash(input.role,input.content)}}countExistingReplayRowsAtTimestampForIdentity(conversationId,role,identityHash,content,createdAt){const row=this.db.prepare(`SELECT COUNT(*) AS count
|
|
612
|
+
FROM messages AS m
|
|
613
|
+
WHERE m.conversation_id = ?
|
|
614
|
+
AND m.role = ?
|
|
615
|
+
AND m.identity_hash = ?
|
|
616
|
+
AND m.content = ?
|
|
617
|
+
AND m.created_at = ?
|
|
618
|
+
AND length(m.content) > 0
|
|
619
|
+
AND EXISTS (
|
|
620
|
+
SELECT 1
|
|
621
|
+
FROM messages AS prior
|
|
622
|
+
WHERE prior.conversation_id = m.conversation_id
|
|
623
|
+
AND prior.identity_hash = m.identity_hash
|
|
624
|
+
AND prior.role = m.role
|
|
625
|
+
AND prior.content = m.content
|
|
626
|
+
AND prior.created_at < m.created_at
|
|
627
|
+
)`).get(conversationId,role,identityHash,content,createdAt);return row?.count??0}countExistingReplayRowsAtTimestampForRole(conversationId,role,createdAt){const row=this.db.prepare(`SELECT COUNT(*) AS count
|
|
628
|
+
FROM messages AS m
|
|
629
|
+
WHERE m.conversation_id = ?
|
|
630
|
+
AND m.role = ?
|
|
631
|
+
AND m.created_at = ?
|
|
632
|
+
AND length(m.content) > 0
|
|
633
|
+
AND EXISTS (
|
|
634
|
+
SELECT 1
|
|
635
|
+
FROM messages AS prior
|
|
636
|
+
WHERE prior.conversation_id = m.conversation_id
|
|
637
|
+
AND prior.identity_hash = m.identity_hash
|
|
638
|
+
AND prior.role = m.role
|
|
639
|
+
AND prior.content = m.content
|
|
640
|
+
AND prior.created_at < m.created_at
|
|
641
|
+
)`).get(conversationId,role,createdAt);return row?.count??0}assertNoReplayTimestampFlood(inputs){if(inputs.length===0){return}const replicatedByGroup=new Map;for(const input of inputs){if(input.content.length===0){continue}const priorCount=this.countMessagesByIdentityBeforeTimestampSync({conversationId:input.conversationId,role:input.role,content:input.content,beforeCreatedAt:input.createdAt});if(priorCount===0){continue}const key=input.role==="user"?JSON.stringify([input.conversationId,input.role,input.createdAt]):JSON.stringify([input.conversationId,input.role,input.identityHash,input.content,input.createdAt]);const group=replicatedByGroup.get(key);if(group){group.candidateCount+=1}else if(input.role==="user"){replicatedByGroup.set(key,{scope:"external",conversationId:input.conversationId,role:input.role,createdAt:input.createdAt,candidateCount:1})}else{replicatedByGroup.set(key,{scope:"internal",conversationId:input.conversationId,role:input.role,content:input.content,identityHash:input.identityHash,createdAt:input.createdAt,candidateCount:1})}}for(const group of replicatedByGroup.values()){const existingCount=group.scope==="external"?this.countExistingReplayRowsAtTimestampForRole(group.conversationId,group.role,group.createdAt):this.countExistingReplayRowsAtTimestampForIdentity(group.conversationId,group.role,group.identityHash,group.content,group.createdAt);const replicatedCount=existingCount+group.candidateCount;const threshold=this.replayFloodThresholdForRole(group.role);if(replicatedCount>=threshold){throw new Error(`[lcm] refused replay-like message batch: conversation=${group.conversationId} role=${group.role} createdAt=${group.createdAt} replicatedRows=${replicatedCount} threshold=${threshold}`)}}}replayFloodThresholdForRole(role){if(role==="user")return this.replayFloodThresholdExternal;return this.replayFloodThresholdInternal}deleteMessageFromFullText(messageId){if(!this.fts5Available){return}try{this.db.prepare(`DELETE FROM messages_fts WHERE rowid = ?`).run(messageId)}catch{}}searchFullText(query,limit,conversationId,conversationIds,since,before,sort){const where=["messages_fts MATCH ?"];const args=[sanitizeFts5Query(query)];appendConversationScopeConstraint({where,args,columnExpr:"m.conversation_id",conversationId,conversationIds});if(since){where.push("julianday(m.created_at) >= julianday(?)");args.push(since.toISOString())}if(before){where.push("julianday(m.created_at) < julianday(?)");args.push(before.toISOString())}args.push(limit);const orderBy=buildFtsOrderBy(sort,"m.created_at");const sql=`SELECT
|
|
642
|
+
m.message_id,
|
|
643
|
+
m.conversation_id,
|
|
644
|
+
m.role,
|
|
645
|
+
snippet(messages_fts, 0, '', '', '...', 32) AS snippet,
|
|
646
|
+
rank,
|
|
647
|
+
m.created_at
|
|
648
|
+
FROM messages_fts
|
|
649
|
+
JOIN messages m ON m.message_id = messages_fts.rowid
|
|
650
|
+
WHERE ${where.join(" AND ")}
|
|
651
|
+
ORDER BY ${orderBy}
|
|
652
|
+
LIMIT ?`;const rows=this.db.prepare(sql).all(...args);return rows.map(toSearchResult)}searchLike(query,limit,conversationId,conversationIds,since,before){const plan=buildLikeSearchPlan("content",query);if(plan.terms.length===0){return[]}const where=[...plan.where];const args=[...plan.args];appendConversationScopeConstraint({where,args,columnExpr:"conversation_id",conversationId,conversationIds});if(since){where.push("julianday(created_at) >= julianday(?)");args.push(since.toISOString())}if(before){where.push("julianday(created_at) < julianday(?)");args.push(before.toISOString())}args.push(limit);const whereClause=where.length>0?`WHERE ${where.join(" AND ")}`:"";const rows=this.db.prepare(`SELECT message_id, conversation_id, seq, role, content, token_count, created_at, large_content
|
|
653
|
+
FROM messages
|
|
654
|
+
${whereClause}
|
|
655
|
+
ORDER BY created_at DESC
|
|
656
|
+
LIMIT ?`).all(...args);return rows.map(row=>{const normalizedContent=normalizeMessageContentForFullTextIndex(row.content)??row.content;const haystack=normalizedContent.toLowerCase();const matchesAllTerms=plan.terms.every(term=>haystack.includes(term));if(!matchesAllTerms){return null}return{messageId:row.message_id,conversationId:row.conversation_id,role:row.role,snippet:createFallbackSnippet(normalizedContent,plan.terms),createdAt:parseUtcTimestamp(row.created_at),rank:0}}).filter(row=>row!==null)}searchRegex(pattern,limit,conversationId,conversationIds,since,before){if(pattern.length>500||/(\+|\*|\?)\)(\+|\*|\?|\{\d)/.test(pattern)){return[]}let re;try{re=new RegExp(pattern)}catch{return[]}const where=[];const args=[];appendConversationScopeConstraint({where,args,columnExpr:"conversation_id",conversationId,conversationIds});if(since){where.push("julianday(created_at) >= julianday(?)");args.push(since.toISOString())}if(before){where.push("julianday(created_at) < julianday(?)");args.push(before.toISOString())}const whereClause=where.length>0?`WHERE ${where.join(" AND ")}`:"";const rows=this.db.prepare(`SELECT message_id, conversation_id, seq, role, content, token_count, created_at, large_content
|
|
657
|
+
FROM messages
|
|
658
|
+
${whereClause}
|
|
659
|
+
ORDER BY created_at DESC`).all(...args);const MAX_ROW_SCAN=1e4;const results=[];let scanned=0;for(const row of rows){if(results.length>=limit||scanned>=MAX_ROW_SCAN){break}scanned++;const match=re.exec(row.content);if(match){results.push({messageId:row.message_id,conversationId:row.conversation_id,role:row.role,snippet:match[0],createdAt:parseUtcTimestamp(row.created_at),rank:0})}}return results}};import{open,realpath}from"node:fs/promises";import{resolve as resolvePath,sep as pathSep}from"node:path";var SUMMARY_SEARCH_TIME_EXPR="COALESCE(s.latest_at, s.created_at)";var SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED="COALESCE(latest_at, created_at)";var CJK_QUERY_SEGMENT_RE=/[\u2E80-\u9FFF\u3400-\u4DBF\uF900-\uFAFF\uAC00-\uD7AF\u3040-\u309F\u30A0-\u30FF]+/g;var LATIN_QUERY_TOKEN_RE=/[a-zA-Z0-9][\w./-]*/g;function toSummaryRecord(row){let fileIds=[];try{fileIds=JSON.parse(row.file_ids)}catch{}return{summaryId:row.summary_id,conversationId:row.conversation_id,kind:row.kind,depth:row.depth,content:row.content,tokenCount:row.token_count,fileIds,earliestAt:parseUtcTimestampOrNull(row.earliest_at),latestAt:parseUtcTimestampOrNull(row.latest_at),descendantCount:typeof row.descendant_count==="number"&&Number.isFinite(row.descendant_count)&&row.descendant_count>=0?Math.floor(row.descendant_count):0,descendantTokenCount:typeof row.descendant_token_count==="number"&&Number.isFinite(row.descendant_token_count)&&row.descendant_token_count>=0?Math.floor(row.descendant_token_count):0,sourceMessageTokenCount:typeof row.source_message_token_count==="number"&&Number.isFinite(row.source_message_token_count)&&row.source_message_token_count>=0?Math.floor(row.source_message_token_count):0,model:typeof row.model==="string"?row.model:"unknown",createdAt:parseUtcTimestamp(row.created_at)}}function toContextItemRecord(row){return{conversationId:row.conversation_id,ordinal:row.ordinal,itemType:row.item_type,messageId:row.message_id,summaryId:row.summary_id,createdAt:parseUtcTimestamp(row.created_at)}}function toSearchResult2(row){return{summaryId:row.summary_id,conversationId:row.conversation_id,kind:row.kind,snippet:row.snippet,createdAt:parseUtcTimestamp(row.created_at),rank:row.rank}}function toLargeFileRecord(row){return{fileId:row.file_id,conversationId:row.conversation_id,fileName:row.file_name,mimeType:row.mime_type,byteSize:row.byte_size,storageUri:row.storage_uri,explorationSummary:row.exploration_summary,createdAt:parseUtcTimestamp(row.created_at)}}function toConversationBootstrapStateRecord(row){return{conversationId:row.conversation_id,sessionFilePath:row.session_file_path,lastSeenSize:row.last_seen_size,lastSeenMtimeMs:row.last_seen_mtime_ms,lastProcessedOffset:row.last_processed_offset,lastProcessedEntryHash:row.last_processed_entry_hash,sessionHeaderId:row.session_header_id??null,lastProcessedEntryId:row.last_processed_entry_id??null,forkBounded:row.fork_bounded===1,forkSourceMessageCount:typeof row.fork_source_message_count==="number"&&Number.isFinite(row.fork_source_message_count)&&row.fork_source_message_count>=0?Math.floor(row.fork_source_message_count):0,updatedAt:parseUtcTimestamp(row.updated_at)}}function toTranscriptGcCandidateRecord(row){if(typeof row.tool_call_id!=="string"||row.tool_call_id.length===0){return null}let metadata=null;try{metadata=typeof row.metadata==="string"&&row.metadata.length>0?JSON.parse(row.metadata):null}catch{metadata=null}if(!metadata||metadata.toolOutputExternalized!==true){return null}return{messageId:row.message_id,conversationId:row.conversation_id,seq:row.seq,toolCallId:row.tool_call_id,toolName:row.tool_name,externalizedFileId:typeof metadata.externalizedFileId==="string"?metadata.externalizedFileId:null,originalByteSize:typeof metadata.originalByteSize==="number"&&Number.isFinite(metadata.originalByteSize)?Math.max(0,Math.floor(metadata.originalByteSize)):null}}var SummaryStore=class{constructor(db,options){this.db=db;this.fts5Available=options?.fts5Available??true}db;fts5Available;async insertSummary(input){const fileIds=JSON.stringify(input.fileIds??[]);const earliestAt=input.earliestAt instanceof Date?input.earliestAt.toISOString():null;const latestAt=input.latestAt instanceof Date?input.latestAt.toISOString():null;const descendantCount=typeof input.descendantCount==="number"&&Number.isFinite(input.descendantCount)&&input.descendantCount>=0?Math.floor(input.descendantCount):0;const descendantTokenCount=typeof input.descendantTokenCount==="number"&&Number.isFinite(input.descendantTokenCount)&&input.descendantTokenCount>=0?Math.floor(input.descendantTokenCount):0;const sourceMessageTokenCount=typeof input.sourceMessageTokenCount==="number"&&Number.isFinite(input.sourceMessageTokenCount)&&input.sourceMessageTokenCount>=0?Math.floor(input.sourceMessageTokenCount):0;const depth=typeof input.depth==="number"&&Number.isFinite(input.depth)&&input.depth>=0?Math.floor(input.depth):input.kind==="leaf"?0:1;this.db.prepare(`INSERT INTO summaries (
|
|
660
|
+
summary_id,
|
|
661
|
+
conversation_id,
|
|
662
|
+
kind,
|
|
663
|
+
depth,
|
|
664
|
+
content,
|
|
665
|
+
token_count,
|
|
666
|
+
file_ids,
|
|
667
|
+
earliest_at,
|
|
668
|
+
latest_at,
|
|
669
|
+
descendant_count,
|
|
670
|
+
descendant_token_count,
|
|
671
|
+
source_message_token_count,
|
|
672
|
+
model
|
|
673
|
+
)
|
|
674
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(input.summaryId,input.conversationId,input.kind,depth,input.content,input.tokenCount,fileIds,earliestAt,latestAt,descendantCount,descendantTokenCount,sourceMessageTokenCount,input.model??"unknown");const row=this.db.prepare(`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
675
|
+
earliest_at, latest_at, descendant_count, created_at
|
|
676
|
+
, descendant_token_count, source_message_token_count, model
|
|
677
|
+
FROM summaries WHERE summary_id = ?`).get(input.summaryId);if(!this.fts5Available){return toSummaryRecord(row)}try{this.db.prepare(`INSERT INTO summaries_fts(summary_id, content) VALUES (?, ?)`).run(input.summaryId,input.content)}catch{}try{this.db.prepare(`INSERT INTO summaries_fts_cjk(summary_id, content) VALUES (?, ?)`).run(input.summaryId,input.content)}catch{}return toSummaryRecord(row)}async getSummary(summaryId){const row=this.db.prepare(`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
678
|
+
earliest_at, latest_at, descendant_count, created_at
|
|
679
|
+
, descendant_token_count, source_message_token_count, model
|
|
680
|
+
FROM summaries WHERE summary_id = ?`).get(summaryId);return row?toSummaryRecord(row):null}async getSummaryMessageSeqRange(summaryId){const row=this.db.prepare(`WITH RECURSIVE source_summaries(summary_id) AS (
|
|
681
|
+
SELECT ?
|
|
682
|
+
UNION
|
|
683
|
+
SELECT sp.parent_summary_id
|
|
684
|
+
FROM summary_parents sp
|
|
685
|
+
JOIN source_summaries source ON source.summary_id = sp.summary_id
|
|
686
|
+
)
|
|
687
|
+
SELECT MIN(m.seq) AS min_seq,
|
|
688
|
+
MAX(m.seq) AS max_seq
|
|
689
|
+
FROM source_summaries source
|
|
690
|
+
JOIN summary_messages sm ON sm.summary_id = source.summary_id
|
|
691
|
+
JOIN messages m ON m.message_id = sm.message_id
|
|
692
|
+
`).get(summaryId);return{minSeq:row?.min_seq??null,maxSeq:row?.max_seq??null}}async getSummariesByConversation(conversationId){const rows=this.db.prepare(`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
693
|
+
earliest_at, latest_at, descendant_count, created_at
|
|
694
|
+
, descendant_token_count, source_message_token_count, model
|
|
695
|
+
FROM summaries
|
|
696
|
+
WHERE conversation_id = ?
|
|
697
|
+
ORDER BY created_at`).all(conversationId);return rows.map(toSummaryRecord)}async linkSummaryToMessages(summaryId,messageIds){if(messageIds.length===0){return}const stmt=this.db.prepare(`INSERT INTO summary_messages (summary_id, message_id, ordinal)
|
|
698
|
+
VALUES (?, ?, ?)
|
|
699
|
+
ON CONFLICT (summary_id, message_id) DO NOTHING`);for(let idx=0;idx<messageIds.length;idx++){stmt.run(summaryId,messageIds[idx],idx)}}async linkSummaryToParents(summaryId,parentSummaryIds){if(parentSummaryIds.length===0){return}const stmt=this.db.prepare(`INSERT INTO summary_parents (summary_id, parent_summary_id, ordinal)
|
|
700
|
+
VALUES (?, ?, ?)
|
|
701
|
+
ON CONFLICT (summary_id, parent_summary_id) DO NOTHING`);for(let idx=0;idx<parentSummaryIds.length;idx++){stmt.run(summaryId,parentSummaryIds[idx],idx)}}async getSummaryMessages(summaryId){const rows=this.db.prepare(`SELECT message_id FROM summary_messages
|
|
702
|
+
WHERE summary_id = ?
|
|
703
|
+
ORDER BY ordinal`).all(summaryId);return rows.map(r=>r.message_id)}async getConversationMaxSummaryDepth(conversationId){const row=this.db.prepare(`SELECT MAX(depth) AS max_depth
|
|
704
|
+
FROM summaries
|
|
705
|
+
WHERE conversation_id = ?`).get(conversationId);return typeof row?.max_depth==="number"?row.max_depth:null}async getLeafSummaryLinksForMessageIds(conversationId,messageIds){const normalizedMessageIds=Array.from(new Set(messageIds.filter(messageId=>Number.isInteger(messageId)&&messageId>0)));if(normalizedMessageIds.length===0){return[]}const placeholders=normalizedMessageIds.map(()=>"?").join(", ");const rows=this.db.prepare(`SELECT sm.message_id, sm.summary_id
|
|
706
|
+
FROM summary_messages sm
|
|
707
|
+
JOIN summaries s ON s.summary_id = sm.summary_id
|
|
708
|
+
WHERE s.conversation_id = ?
|
|
709
|
+
AND s.kind = 'leaf'
|
|
710
|
+
AND sm.message_id IN (${placeholders})
|
|
711
|
+
ORDER BY sm.ordinal ASC, s.created_at ASC`).all(conversationId,...normalizedMessageIds);const summaryIdsByMessageId=new Map;for(const row of rows){const existing=summaryIdsByMessageId.get(row.message_id)??[];if(!existing.includes(row.summary_id)){existing.push(row.summary_id);summaryIdsByMessageId.set(row.message_id,existing)}}const orderedLinks=[];for(const messageId of normalizedMessageIds){for(const summaryId of summaryIdsByMessageId.get(messageId)??[]){orderedLinks.push({messageId,summaryId})}}return orderedLinks}async listTranscriptGcCandidates(conversationId,options){const limit=typeof options?.limit==="number"&&Number.isFinite(options.limit)&&options.limit>0?Math.max(1,Math.floor(options.limit)):25;const rows=this.db.prepare(`SELECT
|
|
712
|
+
m.message_id,
|
|
713
|
+
m.conversation_id,
|
|
714
|
+
m.seq,
|
|
715
|
+
mp.tool_call_id,
|
|
716
|
+
mp.tool_name,
|
|
717
|
+
mp.metadata
|
|
718
|
+
FROM messages m
|
|
719
|
+
JOIN message_parts mp
|
|
720
|
+
ON mp.message_id = m.message_id
|
|
721
|
+
WHERE m.conversation_id = ?
|
|
722
|
+
AND m.role = 'tool'
|
|
723
|
+
AND mp.part_type = 'tool'
|
|
724
|
+
AND mp.tool_call_id IS NOT NULL
|
|
725
|
+
AND mp.tool_call_id != ''
|
|
726
|
+
AND EXISTS (
|
|
727
|
+
SELECT 1
|
|
728
|
+
FROM summary_messages sm
|
|
729
|
+
WHERE sm.message_id = m.message_id
|
|
730
|
+
)
|
|
731
|
+
AND NOT EXISTS (
|
|
732
|
+
SELECT 1
|
|
733
|
+
FROM context_items ci
|
|
734
|
+
WHERE ci.conversation_id = m.conversation_id
|
|
735
|
+
AND ci.item_type = 'message'
|
|
736
|
+
AND ci.message_id = m.message_id
|
|
737
|
+
)
|
|
738
|
+
ORDER BY m.seq ASC, mp.ordinal ASC`).all(conversationId);const seenMessageIds=new Set;const candidates=[];for(const row of rows){if(seenMessageIds.has(row.message_id)){continue}const candidate=toTranscriptGcCandidateRecord(row);if(!candidate){continue}seenMessageIds.add(candidate.messageId);candidates.push(candidate);if(candidates.length>=limit){break}}return candidates}async getSummaryChildren(parentSummaryId){const rows=this.db.prepare(`SELECT s.summary_id, s.conversation_id, s.kind, s.depth, s.content, s.token_count,
|
|
739
|
+
s.file_ids, s.earliest_at, s.latest_at, s.descendant_count, s.created_at
|
|
740
|
+
, s.descendant_token_count, s.source_message_token_count, s.model
|
|
741
|
+
FROM summaries s
|
|
742
|
+
JOIN summary_parents sp ON sp.summary_id = s.summary_id
|
|
743
|
+
WHERE sp.parent_summary_id = ?
|
|
744
|
+
ORDER BY sp.ordinal`).all(parentSummaryId);return rows.map(toSummaryRecord)}async getSummaryParents(summaryId){const rows=this.db.prepare(`SELECT s.summary_id, s.conversation_id, s.kind, s.depth, s.content, s.token_count,
|
|
745
|
+
s.file_ids, s.earliest_at, s.latest_at, s.descendant_count, s.created_at
|
|
746
|
+
, s.descendant_token_count, s.source_message_token_count, s.model
|
|
747
|
+
FROM summaries s
|
|
748
|
+
JOIN summary_parents sp ON sp.parent_summary_id = s.summary_id
|
|
749
|
+
WHERE sp.summary_id = ?
|
|
750
|
+
ORDER BY sp.ordinal`).all(summaryId);return rows.map(toSummaryRecord)}async getSummarySubtree(summaryId){const rows=this.db.prepare(`WITH RECURSIVE subtree(summary_id, parent_summary_id, depth_from_root, path) AS (
|
|
751
|
+
SELECT ?, NULL, 0, ''
|
|
752
|
+
UNION ALL
|
|
753
|
+
SELECT
|
|
754
|
+
sp.summary_id,
|
|
755
|
+
sp.parent_summary_id,
|
|
756
|
+
subtree.depth_from_root + 1,
|
|
757
|
+
CASE
|
|
758
|
+
WHEN subtree.path = '' THEN printf('%04d', sp.ordinal)
|
|
759
|
+
ELSE subtree.path || '.' || printf('%04d', sp.ordinal)
|
|
760
|
+
END
|
|
761
|
+
FROM summary_parents sp
|
|
762
|
+
JOIN subtree ON sp.parent_summary_id = subtree.summary_id
|
|
763
|
+
)
|
|
764
|
+
SELECT
|
|
765
|
+
s.summary_id,
|
|
766
|
+
s.conversation_id,
|
|
767
|
+
s.kind,
|
|
768
|
+
s.depth,
|
|
769
|
+
s.content,
|
|
770
|
+
s.token_count,
|
|
771
|
+
s.file_ids,
|
|
772
|
+
s.earliest_at,
|
|
773
|
+
s.latest_at,
|
|
774
|
+
s.descendant_count,
|
|
775
|
+
s.descendant_token_count,
|
|
776
|
+
s.source_message_token_count,
|
|
777
|
+
s.model,
|
|
778
|
+
s.created_at,
|
|
779
|
+
subtree.depth_from_root,
|
|
780
|
+
subtree.parent_summary_id,
|
|
781
|
+
subtree.path,
|
|
782
|
+
(
|
|
783
|
+
SELECT COUNT(*) FROM summary_parents sp2
|
|
784
|
+
WHERE sp2.parent_summary_id = s.summary_id
|
|
785
|
+
) AS child_count
|
|
786
|
+
FROM subtree
|
|
787
|
+
JOIN summaries s ON s.summary_id = subtree.summary_id
|
|
788
|
+
ORDER BY subtree.depth_from_root ASC, subtree.path ASC, s.created_at ASC`).all(summaryId);const seen=new Set;const output=[];for(const row of rows){if(seen.has(row.summary_id)){continue}seen.add(row.summary_id);output.push({...toSummaryRecord(row),depthFromRoot:Math.max(0,Math.floor(row.depth_from_root??0)),parentSummaryId:row.parent_summary_id??null,path:typeof row.path==="string"?row.path:"",childCount:typeof row.child_count==="number"&&Number.isFinite(row.child_count)?Math.max(0,Math.floor(row.child_count)):0})}return output}async getContextItems(conversationId){const rows=this.db.prepare(`SELECT conversation_id, ordinal, item_type, message_id, summary_id, created_at
|
|
789
|
+
FROM context_items
|
|
790
|
+
WHERE conversation_id = ?
|
|
791
|
+
ORDER BY ordinal`).all(conversationId);return rows.map(toContextItemRecord)}async getDistinctDepthsInContext(conversationId,options){const maxOrdinalExclusive=options?.maxOrdinalExclusive;const useOrdinalBound=typeof maxOrdinalExclusive==="number"&&Number.isFinite(maxOrdinalExclusive)&&maxOrdinalExclusive!==Infinity;const sql=useOrdinalBound?`SELECT DISTINCT s.depth
|
|
792
|
+
FROM context_items ci
|
|
793
|
+
JOIN summaries s ON s.summary_id = ci.summary_id
|
|
794
|
+
WHERE ci.conversation_id = ?
|
|
795
|
+
AND ci.item_type = 'summary'
|
|
796
|
+
AND ci.ordinal < ?
|
|
797
|
+
ORDER BY s.depth ASC`:`SELECT DISTINCT s.depth
|
|
798
|
+
FROM context_items ci
|
|
799
|
+
JOIN summaries s ON s.summary_id = ci.summary_id
|
|
800
|
+
WHERE ci.conversation_id = ?
|
|
801
|
+
AND ci.item_type = 'summary'
|
|
802
|
+
ORDER BY s.depth ASC`;const rows=useOrdinalBound?this.db.prepare(sql).all(conversationId,Math.floor(maxOrdinalExclusive)):this.db.prepare(sql).all(conversationId);return rows.map(row=>row.depth)}async withTransaction(operation){return withDatabaseTransaction(this.db,"BEGIN",operation)}async pruneForNewSession(conversationId,retainDepth){if(Number.isFinite(retainDepth)&&retainDepth<0){return}this.db.prepare(`DELETE FROM context_items
|
|
803
|
+
WHERE conversation_id = ?
|
|
804
|
+
AND item_type = 'message'`).run(conversationId);if(!Number.isFinite(retainDepth)){this.db.prepare(`DELETE FROM context_items
|
|
805
|
+
WHERE conversation_id = ?
|
|
806
|
+
AND item_type = 'summary'`).run(conversationId);return}this.db.prepare(`DELETE FROM context_items
|
|
807
|
+
WHERE conversation_id = ?
|
|
808
|
+
AND item_type = 'summary'
|
|
809
|
+
AND summary_id IN (
|
|
810
|
+
SELECT summary_id
|
|
811
|
+
FROM summaries
|
|
812
|
+
WHERE conversation_id = ?
|
|
813
|
+
AND depth < ?
|
|
814
|
+
)`).run(conversationId,conversationId,Math.floor(retainDepth))}async appendContextMessage(conversationId,messageId){const row=this.db.prepare(`SELECT COALESCE(MAX(ordinal), -1) AS max_ordinal
|
|
815
|
+
FROM context_items WHERE conversation_id = ?`).get(conversationId);this.db.prepare(`INSERT INTO context_items (conversation_id, ordinal, item_type, message_id)
|
|
816
|
+
VALUES (?, ?, 'message', ?)`).run(conversationId,row.max_ordinal+1,messageId)}async appendContextMessages(conversationId,messageIds){if(messageIds.length===0){return}const row=this.db.prepare(`SELECT COALESCE(MAX(ordinal), -1) AS max_ordinal
|
|
817
|
+
FROM context_items WHERE conversation_id = ?`).get(conversationId);const baseOrdinal=row.max_ordinal+1;const stmt=this.db.prepare(`INSERT INTO context_items (conversation_id, ordinal, item_type, message_id)
|
|
818
|
+
VALUES (?, ?, 'message', ?)`);for(let idx=0;idx<messageIds.length;idx++){stmt.run(conversationId,baseOrdinal+idx,messageIds[idx])}}async appendContextSummary(conversationId,summaryId){const row=this.db.prepare(`SELECT COALESCE(MAX(ordinal), -1) AS max_ordinal
|
|
819
|
+
FROM context_items WHERE conversation_id = ?`).get(conversationId);this.db.prepare(`INSERT INTO context_items (conversation_id, ordinal, item_type, summary_id)
|
|
820
|
+
VALUES (?, ?, 'summary', ?)`).run(conversationId,row.max_ordinal+1,summaryId)}async replaceContextRangeWithSummary(input){await this.withTransaction(()=>{this.replaceContextRangeWithSummaryInTransaction(input)})}replaceContextRangeWithSummaryInTransaction(input){const{conversationId,startOrdinal,endOrdinal,summaryId}=input;this.db.prepare(`DELETE FROM context_items
|
|
821
|
+
WHERE conversation_id = ?
|
|
822
|
+
AND ordinal >= ?
|
|
823
|
+
AND ordinal <= ?`).run(conversationId,startOrdinal,endOrdinal);this.db.prepare(`INSERT INTO context_items (conversation_id, ordinal, item_type, summary_id)
|
|
824
|
+
VALUES (?, ?, 'summary', ?)`).run(conversationId,startOrdinal,summaryId);const items=this.db.prepare(`SELECT ordinal FROM context_items
|
|
825
|
+
WHERE conversation_id = ?
|
|
826
|
+
ORDER BY ordinal`).all(conversationId);if(items.length>0&&items.some((item,i)=>item.ordinal!==i)){const updateStmt=this.db.prepare(`UPDATE context_items SET ordinal = ?
|
|
827
|
+
WHERE conversation_id = ? AND ordinal = ?`);for(let i=0;i<items.length;i++){updateStmt.run(-(i+1),conversationId,items[i].ordinal)}for(let i=0;i<items.length;i++){updateStmt.run(i,conversationId,-(i+1))}}}async getContextTokenCount(conversationId){const row=this.db.prepare(`SELECT COALESCE(SUM(token_count), 0) AS total
|
|
828
|
+
FROM (
|
|
829
|
+
SELECT m.token_count
|
|
830
|
+
FROM context_items ci
|
|
831
|
+
JOIN messages m ON m.message_id = ci.message_id
|
|
832
|
+
WHERE ci.conversation_id = ?
|
|
833
|
+
AND ci.item_type = 'message'
|
|
834
|
+
|
|
835
|
+
UNION ALL
|
|
836
|
+
|
|
837
|
+
SELECT s.token_count
|
|
838
|
+
FROM context_items ci
|
|
839
|
+
JOIN summaries s ON s.summary_id = ci.summary_id
|
|
840
|
+
WHERE ci.conversation_id = ?
|
|
841
|
+
AND ci.item_type = 'summary'
|
|
842
|
+
) sub`).get(conversationId,conversationId);return row?.total??0}async searchSummaries(input){const limit=input.limit??50;if(input.mode==="full_text"){if(containsCjk(input.query)){const cjkSegments=this.extractCjkSegments(input.query);const hasShortCjkSegment=cjkSegments.some(segment=>segment.length<3);if(!hasShortCjkSegment){try{const trigramResults=this.searchCjkTrigram(input.query,limit,input.conversationId,input.conversationIds,input.since,input.before,input.sort);if(trigramResults.length>0){return trigramResults}}catch{}}return this.searchLikeCjk(input.query,limit,input.conversationId,input.conversationIds,input.since,input.before)}if(this.fts5Available){try{return this.searchFullText(input.query,limit,input.conversationId,input.conversationIds,input.since,input.before,input.sort)}catch{return this.searchLike(input.query,limit,input.conversationId,input.conversationIds,input.since,input.before)}}return this.searchLike(input.query,limit,input.conversationId,input.conversationIds,input.since,input.before)}return this.searchRegex(input.query,limit,input.conversationId,input.conversationIds,input.since,input.before)}searchFullText(query,limit,conversationId,conversationIds,since,before,sort){const where=["summaries_fts MATCH ?"];const args=[sanitizeFts5Query(query)];appendConversationScopeConstraint({where,args,columnExpr:"s.conversation_id",conversationId,conversationIds});if(since){where.push(`julianday(${SUMMARY_SEARCH_TIME_EXPR}) >= julianday(?)`);args.push(since.toISOString())}if(before){where.push(`julianday(${SUMMARY_SEARCH_TIME_EXPR}) < julianday(?)`);args.push(before.toISOString())}args.push(limit);const orderBy=buildFtsOrderBy(sort,SUMMARY_SEARCH_TIME_EXPR);const sql=`SELECT
|
|
843
|
+
summaries_fts.summary_id,
|
|
844
|
+
s.conversation_id,
|
|
845
|
+
s.kind,
|
|
846
|
+
snippet(summaries_fts, 1, '', '', '...', 32) AS snippet,
|
|
847
|
+
rank,
|
|
848
|
+
${SUMMARY_SEARCH_TIME_EXPR} AS created_at
|
|
849
|
+
FROM summaries_fts
|
|
850
|
+
JOIN summaries s ON s.summary_id = summaries_fts.summary_id
|
|
851
|
+
WHERE ${where.join(" AND ")}
|
|
852
|
+
ORDER BY ${orderBy}
|
|
853
|
+
LIMIT ?`;const rows=this.db.prepare(sql).all(...args);return rows.map(toSearchResult2)}searchLike(query,limit,conversationId,conversationIds,since,before){const plan=buildLikeSearchPlan("content",query);if(plan.terms.length===0){return[]}const where=[...plan.where];const args=[...plan.args];appendConversationScopeConstraint({where,args,columnExpr:"conversation_id",conversationId,conversationIds});if(since){where.push(`julianday(${SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED}) >= julianday(?)`);args.push(since.toISOString())}if(before){where.push(`julianday(${SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED}) < julianday(?)`);args.push(before.toISOString())}args.push(limit);const whereClause=where.length>0?`WHERE ${where.join(" AND ")}`:"";const rows=this.db.prepare(`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
854
|
+
earliest_at, latest_at, descendant_count, descendant_token_count,
|
|
855
|
+
source_message_token_count, model,
|
|
856
|
+
${SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED} AS created_at
|
|
857
|
+
FROM summaries
|
|
858
|
+
${whereClause}
|
|
859
|
+
ORDER BY ${SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED} DESC
|
|
860
|
+
LIMIT ?`).all(...args);return rows.map(row=>({summaryId:row.summary_id,conversationId:row.conversation_id,kind:row.kind,snippet:createFallbackSnippet(row.content,plan.terms),createdAt:parseUtcTimestamp(row.created_at),rank:0}))}extractCjkSegments(query){return query.match(CJK_QUERY_SEGMENT_RE)??[]}extractLatinTokens(query){const tokens=query.match(LATIN_QUERY_TOKEN_RE)??[];return[...new Set(tokens.map(token=>token.toLowerCase()))]}escapeLikeTerm(term){return term.replace(/([\\%_])/g,"\\$1")}splitCjkChunks(text,size){const chunks=[];for(let i=0;i<=text.length-size;i++){const chunk=text.slice(i,i+size);if(!chunks.includes(chunk)){chunks.push(chunk)}}return chunks}searchCjkTrigram(query,limit,conversationId,conversationIds,since,before,sort){const cjkSegments=this.extractCjkSegments(query).filter(segment=>segment.length>=3);if(cjkSegments.length===0){return[]}const latinTokens=this.extractLatinTokens(query);const cjkGroups=[];for(const segment of cjkSegments){const segmentTerms=segment.length<=4?[segment]:this.splitCjkChunks(segment,4);const groupExpr=[...new Set(segmentTerms)].map(term=>`"${term.replace(/"/g,'""')}"`).join(" OR ");cjkGroups.push(`(${groupExpr})`)}const where=["summaries_fts_cjk MATCH ?"];const args=[cjkGroups.join(" AND ")];for(const token of latinTokens){where.push("LOWER(s.content) LIKE ? ESCAPE '\\'");args.push(`%${this.escapeLikeTerm(token)}%`)}appendConversationScopeConstraint({where,args,columnExpr:"s.conversation_id",conversationId,conversationIds});if(since){where.push(`julianday(${SUMMARY_SEARCH_TIME_EXPR}) >= julianday(?)`);args.push(since.toISOString())}if(before){where.push(`julianday(${SUMMARY_SEARCH_TIME_EXPR}) < julianday(?)`);args.push(before.toISOString())}args.push(limit);const orderBy=buildFtsOrderBy(sort,SUMMARY_SEARCH_TIME_EXPR);const sql=`SELECT
|
|
861
|
+
f.summary_id,
|
|
862
|
+
s.conversation_id,
|
|
863
|
+
s.kind,
|
|
864
|
+
snippet(summaries_fts_cjk, 1, '', '', '...', 32) AS snippet,
|
|
865
|
+
rank,
|
|
866
|
+
${SUMMARY_SEARCH_TIME_EXPR} AS created_at
|
|
867
|
+
FROM summaries_fts_cjk f
|
|
868
|
+
JOIN summaries s ON s.summary_id = f.summary_id
|
|
869
|
+
WHERE ${where.join(" AND ")}
|
|
870
|
+
ORDER BY ${orderBy}
|
|
871
|
+
LIMIT ?`;const rows=this.db.prepare(sql).all(...args);return rows.map(toSearchResult2)}searchLikeCjk(query,limit,conversationId,conversationIds,since,before){const cjkSegments=this.extractCjkSegments(query);const latinTokens=this.extractLatinTokens(query);if(cjkSegments.length===0&&latinTokens.length===0){return[]}const cjkTerms=[];const cjkClauses=[];const cjkArgs=[];for(const segment of cjkSegments){const segmentTerms=segment.length===1?[segment]:segment.length===2?[segment]:this.splitCjkChunks(segment,2);const uniqueTerms=[...new Set(segmentTerms)];cjkTerms.push(...uniqueTerms);cjkClauses.push(`(${uniqueTerms.map(()=>`LOWER(content) LIKE ? ESCAPE '\\'`).join(" OR ")})`);cjkArgs.push(...uniqueTerms.map(term=>`%${this.escapeLikeTerm(term.toLowerCase())}%`))}const latinClauses=latinTokens.map(()=>`LOWER(content) LIKE ? ESCAPE '\\'`);const latinArgs=latinTokens.map(token=>`%${this.escapeLikeTerm(token)}%`);const where=[...cjkClauses,...latinClauses];const args=[...cjkArgs,...latinArgs];appendConversationScopeConstraint({where,args,columnExpr:"conversation_id",conversationId,conversationIds});if(since){where.push(`julianday(${SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED}) >= julianday(?)`);args.push(since.toISOString())}if(before){where.push(`julianday(${SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED}) < julianday(?)`);args.push(before.toISOString())}args.push(limit);const rows=this.db.prepare(`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
872
|
+
earliest_at, latest_at, descendant_count, descendant_token_count,
|
|
873
|
+
source_message_token_count, model,
|
|
874
|
+
${SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED} AS created_at
|
|
875
|
+
FROM summaries
|
|
876
|
+
WHERE ${where.join(" AND ")}
|
|
877
|
+
ORDER BY ${SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED} DESC
|
|
878
|
+
LIMIT ?`).all(...args);const snippetTerms=cjkTerms.length>0?[...new Set([...cjkTerms,...latinTokens])]:latinTokens;return rows.map(row=>({summaryId:row.summary_id,conversationId:row.conversation_id,kind:row.kind,snippet:createFallbackSnippet(row.content,snippetTerms),createdAt:new Date(row.created_at),rank:0}))}searchRegex(pattern,limit,conversationId,conversationIds,since,before){if(pattern.length>500||/(\+|\*|\?)\)(\+|\*|\?|\{\d)/.test(pattern)){return[]}let re;try{re=new RegExp(pattern)}catch{return[]}const where=[];const args=[];appendConversationScopeConstraint({where,args,columnExpr:"conversation_id",conversationId,conversationIds});if(since){where.push(`julianday(${SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED}) >= julianday(?)`);args.push(since.toISOString())}if(before){where.push(`julianday(${SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED}) < julianday(?)`);args.push(before.toISOString())}const whereClause=where.length>0?`WHERE ${where.join(" AND ")}`:"";const rows=this.db.prepare(`SELECT summary_id, conversation_id, kind, depth, content, token_count, file_ids,
|
|
879
|
+
earliest_at, latest_at, descendant_count, descendant_token_count,
|
|
880
|
+
source_message_token_count, model,
|
|
881
|
+
${SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED} AS created_at
|
|
882
|
+
FROM summaries
|
|
883
|
+
${whereClause}
|
|
884
|
+
ORDER BY ${SUMMARY_SEARCH_TIME_EXPR_UNQUALIFIED} DESC`).all(...args);const MAX_ROW_SCAN=1e4;const results=[];let scanned=0;for(const row of rows){if(results.length>=limit||scanned>=MAX_ROW_SCAN){break}scanned++;const match=re.exec(row.content);if(match){results.push({summaryId:row.summary_id,conversationId:row.conversation_id,kind:row.kind,snippet:match[0],createdAt:parseUtcTimestamp(row.created_at),rank:0})}}return results}async insertLargeFile(input){this.db.prepare(`INSERT INTO large_files (file_id, conversation_id, file_name, mime_type, byte_size, storage_uri, exploration_summary)
|
|
885
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`).run(input.fileId,input.conversationId,input.fileName??null,input.mimeType??null,input.byteSize??null,input.storageUri,input.explorationSummary??null);const row=this.db.prepare(`SELECT file_id, conversation_id, file_name, mime_type, byte_size, storage_uri, exploration_summary, created_at
|
|
886
|
+
FROM large_files WHERE file_id = ?`).get(input.fileId);return toLargeFileRecord(row)}async getLargeFile(fileId){const row=this.db.prepare(`SELECT file_id, conversation_id, file_name, mime_type, byte_size, storage_uri, exploration_summary, created_at
|
|
887
|
+
FROM large_files WHERE file_id = ?`).get(fileId);return row?toLargeFileRecord(row):null}async getLargeFilesByConversation(conversationId){const rows=this.db.prepare(`SELECT file_id, conversation_id, file_name, mime_type, byte_size, storage_uri, exploration_summary, created_at
|
|
888
|
+
FROM large_files
|
|
889
|
+
WHERE conversation_id = ?
|
|
890
|
+
ORDER BY created_at`).all(conversationId);return rows.map(toLargeFileRecord)}async largeFileContentEquals(fileId,content,options){return this.largeFileBufferEquals(fileId,Buffer.from(content,"utf8"),options)}async largeFileBufferEquals(fileId,content,options){const byteSize=content.byteLength;const row=this.db.prepare(`SELECT storage_uri
|
|
891
|
+
FROM large_files
|
|
892
|
+
WHERE file_id = ?
|
|
893
|
+
AND byte_size = ?
|
|
894
|
+
LIMIT 1`).get(fileId,byteSize);if(!row){return false}return this.validatedLargeFileBufferEquals(row.storage_uri,content,options)}async getLargeFileContent(fileId,options){const row=this.db.prepare(`SELECT storage_uri
|
|
895
|
+
FROM large_files
|
|
896
|
+
WHERE file_id = ?
|
|
897
|
+
LIMIT 1`).get(fileId);if(!row){return null}return this.readValidatedLargeFileContent(row.storage_uri,options)}async validatedLargeFileContentEquals(storageUri,expectedContent,options){return this.validatedLargeFileBufferEquals(storageUri,Buffer.from(expectedContent,"utf8"),options)}async validatedLargeFileBufferEquals(storageUri,expected,options){try{const file=await this.openValidatedLargeFile(storageUri,options);if(!file){return false}try{const stats=await file.stat();if(!stats.isFile()||stats.size!==expected.length){return false}const buffer=Buffer.allocUnsafe(Math.min(64*1024,Math.max(1,expected.length)));let offset=0;while(offset<expected.length){const length=Math.min(buffer.length,expected.length-offset);const{bytesRead}=await file.read(buffer,0,length,offset);if(bytesRead!==length){return false}if(!buffer.subarray(0,length).equals(expected.subarray(offset,offset+length))){return false}offset+=length}return true}finally{await file.close().catch(()=>void 0)}}catch{return false}}async readValidatedLargeFileContent(storageUri,options){try{const file=await this.openValidatedLargeFile(storageUri,options);if(!file){return null}try{const stats=await file.stat();if(!stats.isFile()||options.maxBytes!=null&&stats.size>options.maxBytes){return null}return await file.readFile({encoding:"utf8"})}finally{await file.close().catch(()=>void 0)}}catch{return null}}async openValidatedLargeFile(storageUri,options){const safeRoot=await realpath(resolvePath(options.largeFilesDir));const realTarget=await realpath(resolvePath(storageUri));if(realTarget!==safeRoot&&!realTarget.startsWith(safeRoot+pathSep)){return null}return open(realTarget,"r")}async getConversationBootstrapState(conversationId){const row=this.db.prepare(`SELECT conversation_id, session_file_path, last_seen_size, last_seen_mtime_ms,
|
|
898
|
+
last_processed_offset, last_processed_entry_hash, session_header_id,
|
|
899
|
+
last_processed_entry_id, fork_bounded,
|
|
900
|
+
fork_source_message_count, updated_at
|
|
901
|
+
FROM conversation_bootstrap_state
|
|
902
|
+
WHERE conversation_id = ?`).get(conversationId);return row?toConversationBootstrapStateRecord(row):null}async upsertConversationBootstrapState(input){this.db.prepare(`INSERT INTO conversation_bootstrap_state (
|
|
903
|
+
conversation_id,
|
|
904
|
+
session_file_path,
|
|
905
|
+
last_seen_size,
|
|
906
|
+
last_seen_mtime_ms,
|
|
907
|
+
last_processed_offset,
|
|
908
|
+
last_processed_entry_hash,
|
|
909
|
+
session_header_id,
|
|
910
|
+
last_processed_entry_id,
|
|
911
|
+
fork_bounded,
|
|
912
|
+
fork_source_message_count
|
|
913
|
+
)
|
|
914
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
915
|
+
ON CONFLICT (conversation_id) DO UPDATE SET
|
|
916
|
+
session_file_path = excluded.session_file_path,
|
|
917
|
+
last_seen_size = excluded.last_seen_size,
|
|
918
|
+
last_seen_mtime_ms = excluded.last_seen_mtime_ms,
|
|
919
|
+
last_processed_offset = excluded.last_processed_offset,
|
|
920
|
+
last_processed_entry_hash = excluded.last_processed_entry_hash,
|
|
921
|
+
session_header_id = COALESCE(
|
|
922
|
+
excluded.session_header_id,
|
|
923
|
+
conversation_bootstrap_state.session_header_id
|
|
924
|
+
),
|
|
925
|
+
last_processed_entry_id = excluded.last_processed_entry_id,
|
|
926
|
+
fork_bounded = CASE
|
|
927
|
+
WHEN excluded.fork_bounded = 1 THEN 1
|
|
928
|
+
WHEN conversation_bootstrap_state.session_file_path != excluded.session_file_path THEN 0
|
|
929
|
+
ELSE conversation_bootstrap_state.fork_bounded
|
|
930
|
+
END,
|
|
931
|
+
fork_source_message_count = CASE
|
|
932
|
+
WHEN excluded.fork_bounded = 1 THEN excluded.fork_source_message_count
|
|
933
|
+
WHEN conversation_bootstrap_state.session_file_path != excluded.session_file_path THEN 0
|
|
934
|
+
ELSE conversation_bootstrap_state.fork_source_message_count
|
|
935
|
+
END,
|
|
936
|
+
updated_at = datetime('now')`).run(input.conversationId,input.sessionFilePath,Math.max(0,Math.floor(input.lastSeenSize)),Math.max(0,Math.floor(input.lastSeenMtimeMs)),Math.max(0,Math.floor(input.lastProcessedOffset)),input.lastProcessedEntryHash??null,input.sessionHeaderId??null,input.lastProcessedEntryId??null,input.forkBounded===true?1:0,Math.max(0,Math.floor(input.forkSourceMessageCount??0)));const row=this.db.prepare(`SELECT conversation_id, session_file_path, last_seen_size, last_seen_mtime_ms,
|
|
937
|
+
last_processed_offset, last_processed_entry_hash, session_header_id,
|
|
938
|
+
last_processed_entry_id, fork_bounded,
|
|
939
|
+
fork_source_message_count, updated_at
|
|
940
|
+
FROM conversation_bootstrap_state
|
|
941
|
+
WHERE conversation_id = ?`).get(input.conversationId);return toConversationBootstrapStateRecord(row)}};function defaultStateDir(){return process.env.OPENCLAW_STATE_DIR?.trim()||join(homedir(),".openclaw")}function defaultDbPath(stateDir=defaultStateDir()){return join(stateDir,"lcm.db")}function expandHomePath(pathValue){if(pathValue==="~"){return homedir()}if(pathValue.startsWith("~/")){return join(homedir(),pathValue.slice(2))}return pathValue}function normalizePathInput(pathValue){return resolve2(expandHomePath(pathValue))}function normalizeLimit(limit){if(limit===void 0){return void 0}if(!Number.isFinite(limit)||limit<0){throw new Error(`Invalid --limit value: ${limit}`)}return Math.floor(limit)}function normalizeSince(since){if(since===void 0){return void 0}const value=since instanceof Date?since:new Date(since);if(!Number.isFinite(value.getTime())){throw new Error(`Invalid --since value: ${String(since)}`)}return value}async function safeStat(file){try{const fileStat=await stat(file);if(!fileStat.isFile()){return null}await access(file,constants.R_OK);return{size:fileStat.size,mtimeMs:fileStat.mtimeMs}}catch{return null}}async function listJsonlFilesInDirectory(dir){try{const entries=await readdir(dir,{withFileTypes:true,encoding:"utf8"});return entries.filter(entry=>entry.isFile()&&entry.name.endsWith(".jsonl")).map(entry=>join(dir,entry.name))}catch{return[]}}async function listDefaultSessionFiles(stateDir){const agentsDir=join(stateDir,"agents");let agents;try{agents=await readdir(agentsDir,{withFileTypes:true,encoding:"utf8"})}catch{return[]}const files=[];for(const agent of agents){if(!agent.isDirectory()){continue}files.push(...await listJsonlFilesInDirectory(join(agentsDir,agent.name,"sessions")))}return files}async function discoverSessionFiles(options={}){const stateDir=normalizePathInput(options.stateDir??defaultStateDir());const since=normalizeSince(options.since);const limit=normalizeLimit(options.limit);if(limit===0){return[]}const discovered=new Set;for(const file of options.files??[]){discovered.add(normalizePathInput(file))}for(const sessionDir of options.sessionDirs??[]){for(const file of await listJsonlFilesInDirectory(normalizePathInput(sessionDir))){discovered.add(resolve2(file))}}if((options.files?.length??0)===0&&(options.sessionDirs?.length??0)===0){for(const file of await listDefaultSessionFiles(stateDir)){discovered.add(resolve2(file))}}const files=[...discovered].sort((left,right)=>left.localeCompare(right));const filtered=[];for(const file of files){const fileStat=await safeStat(file);if(!fileStat){filtered.push(file);continue}if(since&&fileStat.mtimeMs<since.getTime()){continue}filtered.push(file);if(limit!==void 0&&filtered.length>=limit){break}}return filtered}async function prepareSessionFile(file){const fileStat=await safeStat(file);if(!fileStat){return{file,status:"error",sessionId:null,candidateMessages:0,importedMessages:0,skippedMessages:0,reason:"unreadable-file",warnings:[],error:"File does not exist, is not readable, or is not a regular file."}}const[header,rawMessages]=await Promise.all([readTranscriptHeader(file),readLeafPathMessages(file)]);const messages=filterPersistableMessages(rawMessages);const sessionId=header.sessionHeaderId??basename(file,extname(file));if(messages.length===0){return{file,status:"skipped",sessionId,candidateMessages:0,importedMessages:0,skippedMessages:0,reason:"no-persistable-messages",warnings:[`No persistable messages were found in ${file}.`]}}return{file,sessionId,sessionHeaderId:header.sessionHeaderId,messages,stat:fileStat}}function isPreparedSessionFile(value){return"messages"in value}async function createDatabaseBackup(dbPath){try{await access(dbPath,constants.R_OK)}catch{return null}const timestamp=new Date().toISOString().replace(/[-:.]/g,"");const backupPath=`${dbPath}.migrate-sessions-${timestamp}.bak`;await mkdir(dirname2(backupPath),{recursive:true});const source=new DatabaseSync2(dbPath,{readOnly:true});try{source.exec("PRAGMA busy_timeout = 30000");source.exec(`VACUUM INTO ${sqliteStringLiteral(backupPath)}`)}finally{source.close()}return backupPath}function sqliteStringLiteral(value){return`'${value.replace(/'/g,"''")}'`}function toImportableMessages(messages){return messages.map(message=>({message,stored:toStoredMessage(message),transcriptEntryId:getTranscriptEntryId(message)}))}function buildDryRunResult(prepared){if(!isPreparedSessionFile(prepared)){return prepared}return{file:prepared.file,status:"would-import",sessionId:prepared.sessionId,candidateMessages:prepared.messages.length,importedMessages:0,skippedMessages:0,warnings:[]}}async function importPreparedFile(db,conversationStore,summaryStore,prepared){const importable=toImportableMessages(prepared.messages);const warnings=[];return withDatabaseTransaction(db,"BEGIN IMMEDIATE",async()=>{const conversation=await conversationStore.getOrCreateConversation(prepared.sessionId,{title:`Imported OpenClaw session ${prepared.sessionId}`});const existingCount=await conversationStore.getMessageCount(conversation.conversationId);const hasMissingTranscriptEntryIds=importable.some(entry=>!entry.transcriptEntryId);if(existingCount>0&&hasMissingTranscriptEntryIds){warnings.push(`Conversation ${prepared.sessionId} already has messages but the transcript lacks stable entry ids; skipping to avoid duplicates.`);return{file:prepared.file,status:"skipped",sessionId:prepared.sessionId,candidateMessages:importable.length,importedMessages:0,skippedMessages:importable.length,reason:"existing-conversation-without-transcript-entry-ids",warnings}}const existingEntryIds=existingCount>0?await conversationStore.filterExistingTranscriptEntryIds(conversation.conversationId,importable.map(entry=>entry.transcriptEntryId).filter(entryId=>entryId!==null)):new Set;const seenEntryIds=new Set;const toImport=[];let skippedMessages=0;for(const entry of importable){if(entry.transcriptEntryId){if(existingEntryIds.has(entry.transcriptEntryId)){skippedMessages+=1;continue}if(seenEntryIds.has(entry.transcriptEntryId)){skippedMessages+=1;warnings.push(`Duplicate transcript entry id ${entry.transcriptEntryId} was skipped within ${prepared.file}.`);continue}seenEntryIds.add(entry.transcriptEntryId);if(existingCount>0){const adopted=await conversationStore.adoptTranscriptEntryId(conversation.conversationId,entry.stored.role,entry.stored.content,entry.transcriptEntryId);if(adopted){existingEntryIds.add(entry.transcriptEntryId);skippedMessages+=1;continue}}}toImport.push(entry)}const createdMessages=[];let nextSeq=await conversationStore.getMaxSeq(conversation.conversationId)+1;for(const entry of toImport){const message=await conversationStore.createMessage({conversationId:conversation.conversationId,seq:nextSeq,role:entry.stored.role,content:entry.stored.content,tokenCount:entry.stored.tokenCount,transcriptEntryId:entry.transcriptEntryId,createdAt:resolveTranscriptMessageCreatedAt(entry.message),skipReplayTimestampFloodGuard:true});nextSeq+=1;await conversationStore.createMessageParts(message.messageId,buildMessageParts({sessionId:prepared.sessionId,message:entry.message,fallbackContent:entry.stored.content}));createdMessages.push(message)}await summaryStore.appendContextMessages(conversation.conversationId,createdMessages.map(message=>message.messageId));await conversationStore.markConversationBootstrapped(conversation.conversationId);const lastImportable=importable[importable.length-1]??null;await summaryStore.upsertConversationBootstrapState({conversationId:conversation.conversationId,sessionFilePath:prepared.file,lastSeenSize:prepared.stat.size,lastSeenMtimeMs:prepared.stat.mtimeMs,lastProcessedOffset:prepared.stat.size,lastProcessedEntryHash:createBootstrapEntryHash(lastImportable?.stored??null),sessionHeaderId:prepared.sessionHeaderId,lastProcessedEntryId:lastImportable?.transcriptEntryId??null,forkBounded:false,forkSourceMessageCount:importable.length});if(createdMessages.length===0){return{file:prepared.file,status:"up-to-date",sessionId:prepared.sessionId,candidateMessages:importable.length,importedMessages:0,skippedMessages,warnings}}return{file:prepared.file,status:"imported",sessionId:prepared.sessionId,candidateMessages:importable.length,importedMessages:createdMessages.length,skippedMessages,warnings}})}function summarizeResult(params){const files=params.files;return{apply:params.apply,dbPath:params.dbPath,stateDir:params.stateDir,backupPath:params.backupPath,scannedFiles:files.length,importedFiles:files.filter(file=>file.status==="imported").length,skippedFiles:files.filter(file=>file.status==="skipped"||file.status==="up-to-date").length,errorFiles:files.filter(file=>file.status==="error").length,importedMessages:files.reduce((total,file)=>total+file.importedMessages,0),files}}async function runSessionMigration(options={}){const stateDir=normalizePathInput(options.stateDir??defaultStateDir());const dbPath=normalizePathInput(options.dbPath??defaultDbPath(stateDir));const apply=options.apply===true;const files=await discoverSessionFiles({...options,stateDir});const prepared=await Promise.all(files.map(file=>prepareSessionFile(file)));if(!apply){return summarizeResult({apply,dbPath,stateDir,backupPath:null,files:prepared.map(buildDryRunResult)})}const backupPath=await createDatabaseBackup(dbPath);const db=createLcmDatabaseConnection(dbPath);try{runLcmMigrations(db);const conversationStore=new ConversationStore(db);const summaryStore=new SummaryStore(db);const results=[];for(const file of prepared){if(!isPreparedSessionFile(file)){results.push(file);continue}try{results.push(await importPreparedFile(db,conversationStore,summaryStore,file))}catch(error){results.push({file:file.file,status:"error",sessionId:file.sessionId,candidateMessages:file.messages.length,importedMessages:0,skippedMessages:0,reason:"import-failed",warnings:[],error:error instanceof Error?error.message:String(error)})}}return summarizeResult({apply,dbPath,stateDir,backupPath,files:results})}finally{closeLcmConnection(db)}}function usage(){return["Usage: lossless-claw-migrate-sessions [options]","","Backfill OpenClaw JSONL session files into lcm.db. Dry-run by default.","","Options:"," --db <path> Database path (default: ${OPENCLAW_STATE_DIR:-~/.openclaw}/lcm.db)"," --state-dir <path> OpenClaw state dir (default: ${OPENCLAW_STATE_DIR:-~/.openclaw})"," --sessions-dir <path> Directory containing *.jsonl sessions; repeatable"," --file <path> Import one JSONL file; repeatable"," --apply Write changes after creating a database backup"," --limit <n> Limit scanned files"," --since <iso-date> Include files modified at/after the date"," --json Print machine-readable JSON"," --verbose Print per-file warnings"," -h, --help Show this help"].join("\n")}function requireValue(args,index,flag){const value=args[index+1];if(!value||value.startsWith("--")){throw new Error(`${flag} requires a value.`)}return value}function parseArgs(argv){const options={sessionDirs:[],files:[]};for(let idx=0;idx<argv.length;idx++){const arg=argv[idx];switch(arg){case"--db":options.dbPath=requireValue(argv,idx,arg);idx+=1;break;case"--state-dir":options.stateDir=requireValue(argv,idx,arg);idx+=1;break;case"--sessions-dir":options.sessionDirs.push(requireValue(argv,idx,arg));idx+=1;break;case"--file":options.files.push(requireValue(argv,idx,arg));idx+=1;break;case"--apply":options.apply=true;break;case"--limit":{const value=Number(requireValue(argv,idx,arg));if(!Number.isInteger(value)||value<0){throw new Error("--limit must be a non-negative integer.")}options.limit=value;idx+=1;break}case"--since":options.since=requireValue(argv,idx,arg);idx+=1;break;case"--json":options.json=true;break;case"--verbose":options.verbose=true;break;case"-h":case"--help":options.help=true;break;default:throw new Error(`Unknown option: ${arg}`)}}if(options.sessionDirs?.length===0){delete options.sessionDirs}if(options.files?.length===0){delete options.files}return options}function formatHumanSummary(result,verbose=false){const mode=result.apply?"apply":"dry-run";const lines=[`lossless-claw-migrate-sessions ${mode}`,`state dir: ${result.stateDir}`,`database: ${result.dbPath}`];if(result.backupPath){lines.push(`backup: ${result.backupPath}`)}else if(result.apply){lines.push("backup: skipped (database did not exist yet)")}lines.push(`files: ${result.scannedFiles} scanned, ${result.importedFiles} imported, ${result.skippedFiles} skipped, ${result.errorFiles} errors`,`messages: ${result.importedMessages} imported`);if(!result.apply){lines.push("dry-run only; rerun with --apply to write changes")}const notableFiles=verbose?result.files:result.files.filter(file=>file.status==="error"||file.status==="skipped");for(const file of notableFiles){const reason=file.reason?` (${file.reason})`:"";lines.push(`- ${file.status}${reason}: ${file.file}`);for(const warning of file.warnings){lines.push(` warning: ${warning}`)}if(file.error){lines.push(` error: ${file.error}`)}}return lines.join("\n")}async function main(){const options=parseArgs(process.argv.slice(2));if(options.help){console.log(usage());return}const stateDir=options.stateDir??defaultStateDir();const dbPath=options.dbPath??defaultDbPath(stateDir);const result=await runSessionMigration({...options,stateDir,dbPath});if(options.json){console.log(JSON.stringify(result,null,2))}else{console.log(formatHumanSummary(result,options.verbose))}if(result.errorFiles>0){process.exitCode=1}}main().catch(error=>{console.error(error instanceof Error?error.message:String(error));process.exitCode=1});
|