agentel 0.2.5 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -37
- package/docs/code-reference.md +26 -13
- package/docs/history-source-handling.md +247 -82
- package/docs/release.md +1 -1
- package/package.json +5 -2
- package/src/archive.js +200 -17
- package/src/canonical-events.js +74 -25
- package/src/cli.js +2561 -204
- package/src/config.js +11 -0
- package/src/doctor.js +2 -0
- package/src/importers/claude.js +309 -11
- package/src/importers/gemini.js +2 -1
- package/src/importers/providers.js +22 -0
- package/src/importers.js +2142 -212
- package/src/parser-versions.js +1 -0
- package/src/search.js +417 -176
- package/src/sources.js +1 -0
- package/src/web-export-instructions.js +79 -0
package/src/archive.js
CHANGED
|
@@ -452,6 +452,8 @@ function hasStructuredMetadata(metadata) {
|
|
|
452
452
|
(Array.isArray(metadata?.toolCalls) && metadata.toolCalls.length) ||
|
|
453
453
|
metadata?.toolResult ||
|
|
454
454
|
metadata?.eventType ||
|
|
455
|
+
(Array.isArray(metadata?.attachments) && metadata.attachments.length) ||
|
|
456
|
+
(Array.isArray(metadata?.assetPointers) && metadata.assetPointers.length) ||
|
|
455
457
|
metadata?.parserVersion
|
|
456
458
|
);
|
|
457
459
|
}
|
|
@@ -541,15 +543,76 @@ function hydrateSessionMetadata(file, metadata) {
|
|
|
541
543
|
return { ...metadata, conversationPath, metadataPath: file, transcriptPath, eventPath, viewPath, rawPath };
|
|
542
544
|
}
|
|
543
545
|
|
|
546
|
+
// TTL cache for the archive walk. The web viewer typically fires several
|
|
547
|
+
// list-derived endpoints (/api/recent, /api/tree, /api/stats) within a single
|
|
548
|
+
// page load; caching the snapshot for a short window lets them share one
|
|
549
|
+
// walk without anyone seeing >1 second of staleness. Tests/imports/CLI paths
|
|
550
|
+
// can invalidate explicitly via `invalidateSessionsSnapshotCache()`.
|
|
551
|
+
const SNAPSHOT_CACHE_TTL_MS = 1000;
|
|
552
|
+
const _snapshotCache = {
|
|
553
|
+
rootKey: "",
|
|
554
|
+
snapshot: null,
|
|
555
|
+
takenAtMs: 0
|
|
556
|
+
};
|
|
557
|
+
|
|
544
558
|
function listSessions(env = process.env) {
|
|
559
|
+
return listSessionsSnapshot(env).sessions;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
function cachedListSessionsSnapshot(env = process.env) {
|
|
563
|
+
const rootKey = path.join(archiveRoot(env), "sessions");
|
|
564
|
+
const now = Date.now();
|
|
565
|
+
if (
|
|
566
|
+
_snapshotCache.snapshot &&
|
|
567
|
+
_snapshotCache.rootKey === rootKey &&
|
|
568
|
+
now - _snapshotCache.takenAtMs < SNAPSHOT_CACHE_TTL_MS
|
|
569
|
+
) {
|
|
570
|
+
return _snapshotCache.snapshot;
|
|
571
|
+
}
|
|
572
|
+
const snapshot = listSessionsSnapshot(env);
|
|
573
|
+
_snapshotCache.rootKey = rootKey;
|
|
574
|
+
_snapshotCache.snapshot = snapshot;
|
|
575
|
+
_snapshotCache.takenAtMs = now;
|
|
576
|
+
return snapshot;
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
function invalidateSessionsSnapshotCache() {
|
|
580
|
+
_snapshotCache.rootKey = "";
|
|
581
|
+
_snapshotCache.snapshot = null;
|
|
582
|
+
_snapshotCache.takenAtMs = 0;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/**
|
|
586
|
+
* Walk the archive once and return both the hydrated session list and a
|
|
587
|
+
* fingerprint summarizing the snapshot. Callers that need an HTTP etag (recent,
|
|
588
|
+
* tree, repo-sessions, stats) can use the fingerprint without re-walking; the
|
|
589
|
+
* fingerprint changes iff some session's metadata mtime/size changed or the
|
|
590
|
+
* count changed (added/removed sessions).
|
|
591
|
+
*
|
|
592
|
+
* The fingerprint deliberately summarizes via XOR over per-file mtime/size
|
|
593
|
+
* values rather than concatenating sorted file paths — XOR is order-independent
|
|
594
|
+
* and folds the whole archive into a single 16-byte hex string in O(N) time
|
|
595
|
+
* with no allocation.
|
|
596
|
+
*/
|
|
597
|
+
function listSessionsSnapshot(env = process.env) {
|
|
545
598
|
const root = path.join(archiveRoot(env), "sessions");
|
|
546
599
|
const seen = new Set();
|
|
547
600
|
const sessions = [];
|
|
601
|
+
let count = 0;
|
|
602
|
+
// 64-bit XOR accumulators kept as JS BigInt so we never overflow. Slightly
|
|
603
|
+
// slower than Number arithmetic but the cost is dwarfed by fs.statSync.
|
|
604
|
+
let mtimeXor = 0n;
|
|
605
|
+
let sizeXor = 0n;
|
|
606
|
+
let maxMtimeMs = 0;
|
|
548
607
|
walk(root, (file) => {
|
|
549
608
|
if (!file.endsWith(".metadata.json")) return;
|
|
550
609
|
seen.add(file);
|
|
551
610
|
let stat;
|
|
552
611
|
try { stat = fs.statSync(file); } catch { return; }
|
|
612
|
+
count += 1;
|
|
613
|
+
mtimeXor ^= BigInt(Math.trunc(stat.mtimeMs));
|
|
614
|
+
sizeXor ^= BigInt(stat.size);
|
|
615
|
+
if (stat.mtimeMs > maxMtimeMs) maxMtimeMs = stat.mtimeMs;
|
|
553
616
|
const cached = _sessionsIndex.byPath.get(file);
|
|
554
617
|
if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) {
|
|
555
618
|
sessions.push(cached.session);
|
|
@@ -569,7 +632,9 @@ function listSessions(env = process.env) {
|
|
|
569
632
|
_sessionsIndex.byPath.delete(key);
|
|
570
633
|
}
|
|
571
634
|
}
|
|
572
|
-
|
|
635
|
+
sessions.sort((a, b) => String(b.startedAt).localeCompare(String(a.startedAt)));
|
|
636
|
+
const fingerprint = `${count.toString(36)}-${mtimeXor.toString(16)}-${sizeXor.toString(16)}`;
|
|
637
|
+
return { sessions, fingerprint, count, maxMtimeMs };
|
|
573
638
|
}
|
|
574
639
|
|
|
575
640
|
function findSessionById(sessionId, env = process.env) {
|
|
@@ -603,7 +668,9 @@ function computeSessionUsage(messages) {
|
|
|
603
668
|
let inputTokens = 0;
|
|
604
669
|
let outputTokens = 0;
|
|
605
670
|
let cacheInputTokens = 0;
|
|
671
|
+
let countedCacheInputTokens = 0;
|
|
606
672
|
let reasoningOutputTokens = 0;
|
|
673
|
+
let countedReasoningOutputTokens = 0;
|
|
607
674
|
let toolUsePromptTokens = 0;
|
|
608
675
|
let maxUndifferentiatedTotalTokens = 0;
|
|
609
676
|
let totalInputTokens = 0;
|
|
@@ -638,7 +705,9 @@ function computeSessionUsage(messages) {
|
|
|
638
705
|
any = true;
|
|
639
706
|
}
|
|
640
707
|
if (summary.cacheInputTokens > 0) cacheInputTokens += summary.cacheInputTokens;
|
|
708
|
+
if (summary.countedCacheInputTokens > 0) countedCacheInputTokens += summary.countedCacheInputTokens;
|
|
641
709
|
if (summary.reasoningOutputTokens > 0) reasoningOutputTokens += summary.reasoningOutputTokens;
|
|
710
|
+
if (summary.countedReasoningOutputTokens > 0) countedReasoningOutputTokens += summary.countedReasoningOutputTokens;
|
|
642
711
|
if (summary.toolUsePromptTokens > 0) toolUsePromptTokens += summary.toolUsePromptTokens;
|
|
643
712
|
if (!summary.hasSplitTokens && !summary.hasExtraTokens && summary.hasTotalTokens) {
|
|
644
713
|
maxUndifferentiatedTotalTokens = Math.max(maxUndifferentiatedTotalTokens, summary.totalTokens);
|
|
@@ -648,17 +717,17 @@ function computeSessionUsage(messages) {
|
|
|
648
717
|
}
|
|
649
718
|
if (Number.isFinite(totalInputTokens) && totalInputTokens > 0) any = true;
|
|
650
719
|
if (Number.isFinite(totalOutputTokens) && totalOutputTokens > 0) any = true;
|
|
651
|
-
if (reasoningOutputTokens > 0 || toolUsePromptTokens > 0) any = true;
|
|
720
|
+
if (cacheInputTokens > 0 || reasoningOutputTokens > 0 || toolUsePromptTokens > 0) any = true;
|
|
652
721
|
if (!any && maxUndifferentiatedTotalTokens > 0) {
|
|
653
722
|
inputTokens = maxUndifferentiatedTotalTokens;
|
|
654
723
|
any = true;
|
|
655
724
|
}
|
|
656
725
|
const cumulativeTokens = totalInputTokens + totalOutputTokens;
|
|
657
|
-
const splitTotalTokens = inputTokens + outputTokens +
|
|
726
|
+
const splitTotalTokens = inputTokens + outputTokens + countedCacheInputTokens + countedReasoningOutputTokens + toolUsePromptTokens;
|
|
658
727
|
const finalTotalTokens = splitTotalTokens || cumulativeTokens || maxUndifferentiatedTotalTokens;
|
|
659
728
|
if (!any && finalTotalTokens === 0) return null;
|
|
660
729
|
const result = {
|
|
661
|
-
inputTokens: inputTokens || totalInputTokens || (!outputTokens && !reasoningOutputTokens && !toolUsePromptTokens && maxUndifferentiatedTotalTokens ? maxUndifferentiatedTotalTokens : 0),
|
|
730
|
+
inputTokens: inputTokens || totalInputTokens || (!outputTokens && !cacheInputTokens && !reasoningOutputTokens && !toolUsePromptTokens && maxUndifferentiatedTotalTokens ? maxUndifferentiatedTotalTokens : 0),
|
|
662
731
|
outputTokens: outputTokens || totalOutputTokens,
|
|
663
732
|
cacheInputTokens,
|
|
664
733
|
reasoningOutputTokens,
|
|
@@ -667,6 +736,8 @@ function computeSessionUsage(messages) {
|
|
|
667
736
|
totalInputTokens: totalInputTokens || inputTokens,
|
|
668
737
|
totalOutputTokens: totalOutputTokens || outputTokens
|
|
669
738
|
};
|
|
739
|
+
if (reasoningOutputTokens > 0 && countedReasoningOutputTokens === 0) result.reasoningOutputTokensIncludedInOutput = true;
|
|
740
|
+
if (cacheInputTokens > 0 && countedCacheInputTokens === 0) result.cacheInputTokensIncludedInInput = true;
|
|
670
741
|
if (estimatedUsageCount > 0 && actualUsageCount === 0) {
|
|
671
742
|
result.estimated = true;
|
|
672
743
|
if (estimationMethods.size === 1) result.estimationMethod = Array.from(estimationMethods)[0];
|
|
@@ -804,6 +875,7 @@ function messageTokenEstimate(message) {
|
|
|
804
875
|
function usageTokenSummary(usage) {
|
|
805
876
|
if (!usage || typeof usage !== "object") return emptyUsageTokenSummary();
|
|
806
877
|
const estimated = usage.estimated === true || usage.estimated === "true";
|
|
878
|
+
const authoritativeTotalTokens = usage.authoritativeTotalTokens === true || usage.authoritative_total_tokens === true;
|
|
807
879
|
const estimationMethod = typeof usage.estimationMethod === "string" && usage.estimationMethod.trim()
|
|
808
880
|
? usage.estimationMethod.trim()
|
|
809
881
|
: typeof usage.estimation_method === "string" && usage.estimation_method.trim()
|
|
@@ -868,28 +940,46 @@ function usageTokenSummary(usage) {
|
|
|
868
940
|
firstUsageNumber(usage.cachedContentTokenCount, usage.cached_content_token_count),
|
|
869
941
|
firstUsageNumber(usage.cachedTokens, usage.cached_tokens, usage.cacheTokens, usage.cache_tokens, usage.cached)
|
|
870
942
|
);
|
|
943
|
+
const cacheInputTokensIncludedInInput =
|
|
944
|
+
usage.cacheInputTokensIncludedInInput === true ||
|
|
945
|
+
usage.cache_input_tokens_included_in_input === true ||
|
|
946
|
+
usage.cacheReadTokensIncludedInInput === true ||
|
|
947
|
+
usage.cache_read_tokens_included_in_input === true;
|
|
948
|
+
const countedCacheInputTokens = cacheInputTokensIncludedInInput ? 0 : cacheInputTokens;
|
|
871
949
|
const reasoningOutputTokens = sumPositiveTokenNumbers(
|
|
872
950
|
firstUsageNumber(usage.reasoningOutputTokens, usage.reasoning_output_tokens),
|
|
873
951
|
firstUsageNumber(usage.thoughtsTokens, usage.thoughts_tokens, usage.thoughtsTokenCount, usage.thoughts_token_count),
|
|
874
952
|
firstUsageNumber(usage.reasoningTokens, usage.reasoning_tokens, usage.reasoningTokenCount, usage.reasoning_token_count)
|
|
875
953
|
);
|
|
954
|
+
const reasoningOutputTokensIncludedInOutput =
|
|
955
|
+
usage.reasoningOutputTokensIncludedInOutput === true ||
|
|
956
|
+
usage.reasoning_output_tokens_included_in_output === true ||
|
|
957
|
+
usage.reasoningTokensIncludedInOutput === true ||
|
|
958
|
+
usage.reasoning_tokens_included_in_output === true;
|
|
959
|
+
const countedReasoningOutputTokens = reasoningOutputTokensIncludedInOutput ? 0 : reasoningOutputTokens;
|
|
876
960
|
const toolUsePromptTokens = sumPositiveTokenNumbers(
|
|
877
961
|
firstUsageNumber(usage.toolUsePromptTokens, usage.tool_use_prompt_tokens, usage.toolUsePromptTokenCount, usage.tool_use_prompt_token_count)
|
|
878
962
|
);
|
|
879
|
-
const extraTokens =
|
|
963
|
+
const extraTokens = countedCacheInputTokens + countedReasoningOutputTokens + toolUsePromptTokens;
|
|
880
964
|
const splitTokens = inputTokens + outputTokens;
|
|
881
965
|
const cumulativeTokens = totalInputTokens + totalOutputTokens;
|
|
882
966
|
const categoryTokens = splitTokens + extraTokens;
|
|
883
|
-
const totalTokens =
|
|
884
|
-
?
|
|
885
|
-
:
|
|
967
|
+
const totalTokens = authoritativeTotalTokens && explicitTotalTokens
|
|
968
|
+
? explicitTotalTokens
|
|
969
|
+
: explicitTotalTokens || categoryTokens
|
|
970
|
+
? Math.max(explicitTotalTokens, categoryTokens)
|
|
971
|
+
: cumulativeTokens;
|
|
886
972
|
return {
|
|
887
973
|
inputTokens,
|
|
888
974
|
outputTokens,
|
|
889
975
|
totalInputTokens,
|
|
890
976
|
totalOutputTokens,
|
|
891
977
|
cacheInputTokens,
|
|
978
|
+
countedCacheInputTokens,
|
|
979
|
+
cacheInputTokensIncludedInInput,
|
|
892
980
|
reasoningOutputTokens,
|
|
981
|
+
countedReasoningOutputTokens,
|
|
982
|
+
reasoningOutputTokensIncludedInOutput,
|
|
893
983
|
toolUsePromptTokens,
|
|
894
984
|
extraTokens,
|
|
895
985
|
totalTokens,
|
|
@@ -898,6 +988,7 @@ function usageTokenSummary(usage) {
|
|
|
898
988
|
hasExtraTokens: extraTokens > 0,
|
|
899
989
|
hasTotalTokens: explicitTotalTokens > 0,
|
|
900
990
|
hasCumulativeTokens: cumulativeTokens > 0,
|
|
991
|
+
authoritativeTotalTokens,
|
|
901
992
|
estimated,
|
|
902
993
|
estimationMethod
|
|
903
994
|
};
|
|
@@ -910,7 +1001,11 @@ function emptyUsageTokenSummary() {
|
|
|
910
1001
|
totalInputTokens: 0,
|
|
911
1002
|
totalOutputTokens: 0,
|
|
912
1003
|
cacheInputTokens: 0,
|
|
1004
|
+
countedCacheInputTokens: 0,
|
|
1005
|
+
cacheInputTokensIncludedInInput: false,
|
|
913
1006
|
reasoningOutputTokens: 0,
|
|
1007
|
+
countedReasoningOutputTokens: 0,
|
|
1008
|
+
reasoningOutputTokensIncludedInOutput: false,
|
|
914
1009
|
toolUsePromptTokens: 0,
|
|
915
1010
|
extraTokens: 0,
|
|
916
1011
|
totalTokens: 0,
|
|
@@ -919,6 +1014,7 @@ function emptyUsageTokenSummary() {
|
|
|
919
1014
|
hasExtraTokens: false,
|
|
920
1015
|
hasTotalTokens: false,
|
|
921
1016
|
hasCumulativeTokens: false,
|
|
1017
|
+
authoritativeTotalTokens: false,
|
|
922
1018
|
estimated: false,
|
|
923
1019
|
estimationMethod: ""
|
|
924
1020
|
};
|
|
@@ -951,9 +1047,15 @@ function sessionSummaryUsage(sessionSummary) {
|
|
|
951
1047
|
firstUsageNumber(usage.cacheInputTokens, usage.cache_input_tokens),
|
|
952
1048
|
firstUsageNumber(usage.cacheReadTokens, usage.cache_read_tokens)
|
|
953
1049
|
);
|
|
954
|
-
const
|
|
1050
|
+
const cacheInputTokensIncludedInInput =
|
|
1051
|
+
usage.cacheInputTokensIncludedInInput === true ||
|
|
1052
|
+
usage.cache_input_tokens_included_in_input === true ||
|
|
1053
|
+
usage.cacheReadTokensIncludedInInput === true ||
|
|
1054
|
+
usage.cache_read_tokens_included_in_input === true;
|
|
1055
|
+
const countedCacheInputTokens = cacheInputTokensIncludedInInput ? 0 : cacheInputTokens;
|
|
1056
|
+
const totalTokens = positiveTokenNumber(firstUsageNumber(usage.totalTokens, usage.total_tokens, usage.totalTokenCount, usage.total_token_count)) || inputTokens + outputTokens + countedCacheInputTokens;
|
|
955
1057
|
if (!inputTokens && !outputTokens && !cacheInputTokens && !totalTokens) return null;
|
|
956
|
-
|
|
1058
|
+
const result = {
|
|
957
1059
|
inputTokens,
|
|
958
1060
|
outputTokens,
|
|
959
1061
|
cacheInputTokens,
|
|
@@ -963,22 +1065,39 @@ function sessionSummaryUsage(sessionSummary) {
|
|
|
963
1065
|
totalInputTokens: inputTokens,
|
|
964
1066
|
totalOutputTokens: outputTokens
|
|
965
1067
|
};
|
|
1068
|
+
if (usage.authoritativeTotalTokens === true || usage.authoritative_total_tokens === true) result.authoritativeTotalTokens = true;
|
|
1069
|
+
if (cacheInputTokensIncludedInInput) result.cacheInputTokensIncludedInInput = true;
|
|
1070
|
+
return result;
|
|
966
1071
|
}
|
|
967
1072
|
|
|
968
1073
|
function mergeSessionUsage(primary, secondary) {
|
|
969
1074
|
if (!primary && !secondary) return null;
|
|
970
1075
|
if (!primary) return secondary;
|
|
971
1076
|
if (!secondary) return primary;
|
|
1077
|
+
const totalTokens = secondary.authoritativeTotalTokens
|
|
1078
|
+
? (secondary.totalTokens || 0)
|
|
1079
|
+
: primary.authoritativeTotalTokens
|
|
1080
|
+
? (primary.totalTokens || 0)
|
|
1081
|
+
: Math.max(primary.totalTokens || 0, secondary.totalTokens || 0);
|
|
972
1082
|
const result = {
|
|
973
1083
|
inputTokens: primary.inputTokens || secondary.inputTokens || 0,
|
|
974
1084
|
outputTokens: primary.outputTokens || secondary.outputTokens || 0,
|
|
975
1085
|
cacheInputTokens: primary.cacheInputTokens || secondary.cacheInputTokens || 0,
|
|
976
1086
|
reasoningOutputTokens: primary.reasoningOutputTokens || secondary.reasoningOutputTokens || 0,
|
|
977
1087
|
toolUsePromptTokens: primary.toolUsePromptTokens || secondary.toolUsePromptTokens || 0,
|
|
978
|
-
totalTokens
|
|
1088
|
+
totalTokens,
|
|
979
1089
|
totalInputTokens: primary.totalInputTokens || secondary.totalInputTokens || primary.inputTokens || secondary.inputTokens || 0,
|
|
980
1090
|
totalOutputTokens: primary.totalOutputTokens || secondary.totalOutputTokens || primary.outputTokens || secondary.outputTokens || 0
|
|
981
1091
|
};
|
|
1092
|
+
if (primary.authoritativeTotalTokens || secondary.authoritativeTotalTokens) {
|
|
1093
|
+
result.authoritativeTotalTokens = true;
|
|
1094
|
+
}
|
|
1095
|
+
if (primary.cacheInputTokensIncludedInInput || secondary.cacheInputTokensIncludedInInput) {
|
|
1096
|
+
result.cacheInputTokensIncludedInInput = true;
|
|
1097
|
+
}
|
|
1098
|
+
if (primary.reasoningOutputTokensIncludedInOutput || secondary.reasoningOutputTokensIncludedInOutput) {
|
|
1099
|
+
result.reasoningOutputTokensIncludedInOutput = true;
|
|
1100
|
+
}
|
|
982
1101
|
if (primary.estimated && secondary.estimated) {
|
|
983
1102
|
result.estimated = true;
|
|
984
1103
|
if (primary.estimationMethod && primary.estimationMethod === secondary.estimationMethod) {
|
|
@@ -1047,7 +1166,7 @@ function readEvents(sessionOrPath) {
|
|
|
1047
1166
|
function ensureConversationMarkdown(session, env = process.env) {
|
|
1048
1167
|
const conversationPath = session.conversationPath || session.metadataPath?.replace(/\.metadata\.json$/, ".conversation.md");
|
|
1049
1168
|
if (!conversationPath) return "";
|
|
1050
|
-
if (fs.existsSync(conversationPath)) return conversationPath;
|
|
1169
|
+
if (fs.existsSync(conversationPath) && !conversationMarkdownNeedsRefresh(conversationPath, session)) return conversationPath;
|
|
1051
1170
|
const messages = readTranscript(session.transcriptPath);
|
|
1052
1171
|
if (!messages.length) return "";
|
|
1053
1172
|
const events = readEvents(session);
|
|
@@ -1060,7 +1179,18 @@ function ensureConversationMarkdown(session, env = process.env) {
|
|
|
1060
1179
|
return conversationPath;
|
|
1061
1180
|
}
|
|
1062
1181
|
|
|
1063
|
-
|
|
1182
|
+
function conversationMarkdownNeedsRefresh(file, session) {
|
|
1183
|
+
if (session?.provider !== "chatgpt" && session?.sourceType !== "chatgpt-export") return false;
|
|
1184
|
+
let text = "";
|
|
1185
|
+
try {
|
|
1186
|
+
text = fs.readFileSync(file, "utf8");
|
|
1187
|
+
} catch {
|
|
1188
|
+
return false;
|
|
1189
|
+
}
|
|
1190
|
+
return /\uE200[A-Za-z_]*cite\uE202[^\uE201]+\uE201/.test(text);
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
const VIEW_SCHEMA_VERSION = 4;
|
|
1064
1194
|
|
|
1065
1195
|
function sessionViewPathFromMetadata(metadataPath) {
|
|
1066
1196
|
if (!metadataPath) return "";
|
|
@@ -1234,24 +1364,73 @@ function renderConversationMarkdown(session, messages, events = []) {
|
|
|
1234
1364
|
for (const message of displayMessages) {
|
|
1235
1365
|
body.push(`## ${roleLabel(message.role)} - ${message.timestamp || "unknown time"}`);
|
|
1236
1366
|
body.push("");
|
|
1237
|
-
|
|
1238
|
-
|
|
1367
|
+
const content = styleRedactionMarkersForMarkdown(styleChatGptCitationMarkersForMarkdown(String(message.content || "").trim()));
|
|
1368
|
+
if (content) {
|
|
1369
|
+
body.push(content);
|
|
1370
|
+
body.push("");
|
|
1371
|
+
}
|
|
1372
|
+
const attachmentMarkdown = renderMessageAttachmentsMarkdown(message);
|
|
1373
|
+
if (attachmentMarkdown) {
|
|
1374
|
+
body.push(attachmentMarkdown);
|
|
1375
|
+
body.push("");
|
|
1376
|
+
}
|
|
1239
1377
|
for (const event of events.filter((item) => item.messageIndex === message.index && item.kind === "tool.called")) {
|
|
1240
1378
|
body.push(`### Tool Call - ${event.occurredAt || message.timestamp || "unknown time"}`);
|
|
1241
1379
|
body.push("");
|
|
1242
|
-
body.push(styleRedactionMarkersForMarkdown(String(event.body?.text || event.indexed?.summary || "").trim()));
|
|
1380
|
+
body.push(styleRedactionMarkersForMarkdown(styleChatGptCitationMarkersForMarkdown(String(event.body?.text || event.indexed?.summary || "").trim())));
|
|
1243
1381
|
body.push("");
|
|
1244
1382
|
}
|
|
1245
1383
|
for (const event of events.filter((item) => item.messageIndex === message.index && item.kind === "tool.completed")) {
|
|
1246
1384
|
body.push(`### Tool Result - ${event.occurredAt || message.timestamp || "unknown time"}`);
|
|
1247
1385
|
body.push("");
|
|
1248
|
-
body.push(styleRedactionMarkersForMarkdown(String(event.indexed?.summary || event.body?.text || "").trim()));
|
|
1386
|
+
body.push(styleRedactionMarkersForMarkdown(styleChatGptCitationMarkersForMarkdown(String(event.indexed?.summary || event.body?.text || "").trim())));
|
|
1249
1387
|
body.push("");
|
|
1250
1388
|
}
|
|
1251
1389
|
}
|
|
1252
1390
|
return `${frontmatter.concat(body).join("\n").trimEnd()}\n`;
|
|
1253
1391
|
}
|
|
1254
1392
|
|
|
1393
|
+
function styleChatGptCitationMarkersForMarkdown(value) {
|
|
1394
|
+
return String(value || "").replace(/\uE200([A-Za-z_]*cite)\uE202([^\uE201]+)\uE201/g, (_, kind, ref) => {
|
|
1395
|
+
const label = /file/i.test(String(kind || "")) ? "file citation" : "citation";
|
|
1396
|
+
const text = chatGptCitationParts(ref).join(" ");
|
|
1397
|
+
return text ? `[${label}: ${text}]` : `[${label}]`;
|
|
1398
|
+
});
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
function chatGptCitationParts(ref) {
|
|
1402
|
+
return String(ref || "").split("\uE202").map((part) => part.trim()).filter(Boolean);
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
function renderMessageAttachmentsMarkdown(message) {
|
|
1406
|
+
const attachments = Array.isArray(message?.metadata?.attachments) ? message.metadata.attachments : [];
|
|
1407
|
+
const assetPointers = Array.isArray(message?.metadata?.assetPointers) ? message.metadata.assetPointers : [];
|
|
1408
|
+
if (!attachments.length && !assetPointers.length) return "";
|
|
1409
|
+
const lines = ["### Attachments", ""];
|
|
1410
|
+
for (const attachment of attachments) {
|
|
1411
|
+
const name = markdownLine(attachment?.name || attachment?.id || "attachment");
|
|
1412
|
+
const details = [
|
|
1413
|
+
attachment?.mimeType,
|
|
1414
|
+
attachment?.width && attachment?.height ? `${attachment.width}x${attachment.height}` : "",
|
|
1415
|
+
attachment?.size ? `${attachment.size} bytes` : "",
|
|
1416
|
+
attachment?.assetPointer ? `asset ${attachment.assetPointer}` : ""
|
|
1417
|
+
].filter(Boolean).join(", ");
|
|
1418
|
+
lines.push(`- ${name}${details ? ` (${markdownLine(details)})` : ""}`);
|
|
1419
|
+
}
|
|
1420
|
+
const attachedPointers = new Set(attachments.map((attachment) => attachment?.assetPointer).filter(Boolean));
|
|
1421
|
+
for (const pointer of assetPointers) {
|
|
1422
|
+
if (!pointer?.assetPointer || attachedPointers.has(pointer.assetPointer)) continue;
|
|
1423
|
+
const details = [
|
|
1424
|
+
pointer.contentType,
|
|
1425
|
+
pointer.mimeType,
|
|
1426
|
+
pointer.width && pointer.height ? `${pointer.width}x${pointer.height}` : "",
|
|
1427
|
+
pointer.size ? `${pointer.size} bytes` : ""
|
|
1428
|
+
].filter(Boolean).join(", ");
|
|
1429
|
+
lines.push(`- ${markdownLine(pointer.assetPointer)}${details ? ` (${markdownLine(details)})` : ""}`);
|
|
1430
|
+
}
|
|
1431
|
+
return lines.join("\n");
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1255
1434
|
function normalizePrecomputedEvents(events, session) {
|
|
1256
1435
|
return events
|
|
1257
1436
|
.filter((event) => event && typeof event === "object")
|
|
@@ -1370,7 +1549,11 @@ module.exports = {
|
|
|
1370
1549
|
ensureConversationMarkdown,
|
|
1371
1550
|
ensureSessionView,
|
|
1372
1551
|
findSessionById,
|
|
1552
|
+
invalidateSessionsSnapshotCache,
|
|
1553
|
+
isWebChatProvider,
|
|
1373
1554
|
listSessions,
|
|
1555
|
+
listSessionsSnapshot,
|
|
1556
|
+
cachedListSessionsSnapshot,
|
|
1374
1557
|
normalizeMessages,
|
|
1375
1558
|
objectPathForSession,
|
|
1376
1559
|
readEvents,
|
package/src/canonical-events.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
const crypto = require("crypto");
|
|
4
4
|
const { parserVersionForSource } = require("./parser-versions");
|
|
5
5
|
|
|
6
|
-
const CANONICAL_EVENT_SCHEMA_VERSION = "agentlog.events.
|
|
6
|
+
const CANONICAL_EVENT_SCHEMA_VERSION = "agentlog.events.v2";
|
|
7
7
|
|
|
8
8
|
const EVENT_KINDS = {
|
|
9
9
|
SESSION_STARTED: "session.started",
|
|
@@ -93,6 +93,10 @@ const TOOL_CATEGORY_DEFINITIONS = [
|
|
|
93
93
|
|
|
94
94
|
function normalizeSessionEvents(session, messages, options = {}) {
|
|
95
95
|
const parserVersion = options.parserVersion ?? session.parserVersion ?? parserVersionForSource(session.sourceType);
|
|
96
|
+
const state = {
|
|
97
|
+
toolCallEventIdsByKey: new Map(),
|
|
98
|
+
unmatchedToolCallEventIds: []
|
|
99
|
+
};
|
|
96
100
|
const events = [
|
|
97
101
|
baseEvent(session, {
|
|
98
102
|
messageIndex: -1,
|
|
@@ -114,7 +118,7 @@ function normalizeSessionEvents(session, messages, options = {}) {
|
|
|
114
118
|
})
|
|
115
119
|
];
|
|
116
120
|
for (let index = 0; index < messages.length; index++) {
|
|
117
|
-
events.push(...messageToCanonicalEvents(messages[index], session, { ...options, parserVersion, messageIndex: index }));
|
|
121
|
+
events.push(...messageToCanonicalEvents(messages[index], session, { ...options, parserVersion, messageIndex: index, state }));
|
|
118
122
|
}
|
|
119
123
|
return events;
|
|
120
124
|
}
|
|
@@ -174,31 +178,32 @@ function messageToCanonicalEvents(message, session, options = {}) {
|
|
|
174
178
|
const toolCall = normalizeToolCall(toolCalls[index], metadata.provider || session.provider);
|
|
175
179
|
if (!toolCall) continue;
|
|
176
180
|
const rendered = renderToolCallText(toolCall);
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
);
|
|
181
|
+
const event = baseEvent(session, {
|
|
182
|
+
messageIndex,
|
|
183
|
+
ordinal: index + 1,
|
|
184
|
+
kind: EVENT_KINDS.TOOL_CALLED,
|
|
185
|
+
occurredAt,
|
|
186
|
+
parserVersion,
|
|
187
|
+
role: "assistant",
|
|
188
|
+
parentEventId: responseEventId,
|
|
189
|
+
indexed: {
|
|
190
|
+
title: toolCall.title || toolCall.displayName || titleCaseToolName(toolCall.name || "tool"),
|
|
191
|
+
summary: summarize(rendered || toolCall.rawInputSummary || toolCall.argument || ""),
|
|
192
|
+
toolName: toolCall.name || toolCall.displayName || "",
|
|
193
|
+
toolCategory: toolCall.category || "",
|
|
194
|
+
toolIcon: toolCall.icon || "",
|
|
195
|
+
target: toolCall.target || "",
|
|
196
|
+
status: toolCall.status || "tool_call"
|
|
197
|
+
},
|
|
198
|
+
body: { text: rendered, toolCall, toolResult: null }
|
|
199
|
+
});
|
|
200
|
+
rememberToolCallEvent(options.state, toolCall, event.eventId);
|
|
201
|
+
events.push(event);
|
|
198
202
|
}
|
|
199
203
|
|
|
200
204
|
const toolResult = metadata.toolResult ? normalizeToolResult(metadata.toolResult, metadata.provider || session.provider) : role === "tool" ? normalizeToolResult(content, metadata.provider || session.provider) : null;
|
|
201
205
|
if (toolResult) {
|
|
206
|
+
const parentEventId = consumeToolCallEventId(options.state, toolResult);
|
|
202
207
|
events.push(
|
|
203
208
|
baseEvent(session, {
|
|
204
209
|
messageIndex,
|
|
@@ -210,11 +215,12 @@ function messageToCanonicalEvents(message, session, options = {}) {
|
|
|
210
215
|
indexed: {
|
|
211
216
|
title: toolResult.title || toolResult.kind || "Tool result",
|
|
212
217
|
summary: summarize(toolResult.summary || toolResult.output || content),
|
|
213
|
-
toolName: toolResult.kind || "",
|
|
218
|
+
toolName: toolResult.name || toolResult.kind || "",
|
|
214
219
|
toolCategory: toolResult.category || "",
|
|
215
220
|
toolIcon: toolResult.icon || "",
|
|
216
221
|
status: toolResult.status || "completed"
|
|
217
222
|
},
|
|
223
|
+
parentEventId,
|
|
218
224
|
body: { text: toolResult.output || content, toolCall: null, toolResult }
|
|
219
225
|
})
|
|
220
226
|
);
|
|
@@ -223,6 +229,46 @@ function messageToCanonicalEvents(message, session, options = {}) {
|
|
|
223
229
|
return events;
|
|
224
230
|
}
|
|
225
231
|
|
|
232
|
+
function rememberToolCallEvent(state, toolCall, eventId) {
|
|
233
|
+
if (!state || !eventId) return;
|
|
234
|
+
for (const key of toolPairKeys(toolCall)) {
|
|
235
|
+
if (!state.toolCallEventIdsByKey.has(key)) state.toolCallEventIdsByKey.set(key, []);
|
|
236
|
+
state.toolCallEventIdsByKey.get(key).push(eventId);
|
|
237
|
+
}
|
|
238
|
+
state.unmatchedToolCallEventIds.push(eventId);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function consumeToolCallEventId(state, toolResult) {
|
|
242
|
+
if (!state) return "";
|
|
243
|
+
for (const key of toolPairKeys(toolResult)) {
|
|
244
|
+
const values = state.toolCallEventIdsByKey.get(key);
|
|
245
|
+
while (values && values.length) {
|
|
246
|
+
const eventId = values.shift();
|
|
247
|
+
if (consumeUnmatchedToolEventId(state, eventId)) return eventId;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
return state.unmatchedToolCallEventIds.shift() || "";
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function consumeUnmatchedToolEventId(state, eventId) {
|
|
254
|
+
const index = state.unmatchedToolCallEventIds.indexOf(eventId);
|
|
255
|
+
if (index < 0) return false;
|
|
256
|
+
state.unmatchedToolCallEventIds.splice(index, 1);
|
|
257
|
+
return true;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function toolPairKeys(tool) {
|
|
261
|
+
const keys = new Set();
|
|
262
|
+
for (const value of [tool?.id, tool?.callId, tool?.toolCallId, tool?.tool_call_id]) {
|
|
263
|
+
if (value) keys.add(`id:${String(value).trim()}`);
|
|
264
|
+
}
|
|
265
|
+
for (const value of [tool?.name, tool?.toolName, tool?.displayName, tool?.kind]) {
|
|
266
|
+
const key = normalizeToolKey(value);
|
|
267
|
+
if (key) keys.add(`name:${key}`);
|
|
268
|
+
}
|
|
269
|
+
return [...keys];
|
|
270
|
+
}
|
|
271
|
+
|
|
226
272
|
function baseEvent(session, options) {
|
|
227
273
|
const body = options.body || {};
|
|
228
274
|
const indexed = {
|
|
@@ -371,7 +417,7 @@ function normalizeToolCall(raw, provider = "") {
|
|
|
371
417
|
const target = firstString(raw.target, targetFromArguments(parsedArguments));
|
|
372
418
|
return {
|
|
373
419
|
provider: firstString(raw.provider, provider),
|
|
374
|
-
id: firstString(raw.id, raw.callId, raw.tool_call_id) || undefined,
|
|
420
|
+
id: firstString(raw.id, raw.callId, raw.call_id, raw.toolCallId, raw.tool_call_id, raw.toolUseId, raw.tool_use_id) || undefined,
|
|
375
421
|
name,
|
|
376
422
|
displayName,
|
|
377
423
|
rawCategory: rawCategory || undefined,
|
|
@@ -423,6 +469,7 @@ function normalizeToolResult(raw, provider = "") {
|
|
|
423
469
|
const classification = classifyTool("tool_output");
|
|
424
470
|
return {
|
|
425
471
|
provider,
|
|
472
|
+
id: undefined,
|
|
426
473
|
kind: "Tool output",
|
|
427
474
|
title: "Tool result",
|
|
428
475
|
category: classification.category,
|
|
@@ -443,6 +490,8 @@ function normalizeToolResult(raw, provider = "") {
|
|
|
443
490
|
const classification = classifyTool(firstString(raw.toolName, raw.name, kind), rawCategory);
|
|
444
491
|
return {
|
|
445
492
|
provider: firstString(raw.provider, provider),
|
|
493
|
+
id: firstString(raw.id, raw.callId, raw.call_id, raw.toolCallId, raw.tool_call_id, raw.toolUseId, raw.tool_use_id) || undefined,
|
|
494
|
+
name: firstString(raw.name, raw.toolName, raw.tool_name) || undefined,
|
|
446
495
|
kind,
|
|
447
496
|
title: firstString(raw.title, raw.kind, "Tool result"),
|
|
448
497
|
rawCategory: rawCategory || undefined,
|