bunnyquery 1.8.6 → 1.8.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bunnyquery.css +95 -5
- package/bunnyquery.js +1400 -101
- package/dist/engine.cjs +1018 -78
- package/dist/engine.cjs.map +1 -1
- package/dist/engine.d.mts +506 -163
- package/dist/engine.d.ts +506 -163
- package/dist/engine.mjs +1002 -79
- package/dist/engine.mjs.map +1 -1
- package/package.json +1 -1
- package/src/engine/budget.ts +207 -68
- package/src/engine/config.ts +48 -0
- package/src/engine/errors.ts +38 -0
- package/src/engine/history.ts +546 -6
- package/src/engine/host.ts +7 -0
- package/src/engine/index.ts +15 -1
- package/src/engine/indexing_groups.ts +248 -3
- package/src/engine/office.ts +24 -1
- package/src/engine/prompts/chat_system_prompt.ts +1 -1
- package/src/engine/requests.ts +164 -11
- package/src/engine/session.ts +544 -35
- package/src/widget.css +35 -1
- package/styles/chat.css +60 -4
package/dist/engine.mjs
CHANGED
|
@@ -215,7 +215,8 @@ function isWindowedReadFile(name, mime) {
|
|
|
215
215
|
if (isImageVisionFile(name, mime)) return false;
|
|
216
216
|
return isPagedReadFile(name, mime);
|
|
217
217
|
}
|
|
218
|
-
function composeUserMessage(text, attachmentUrls) {
|
|
218
|
+
function composeUserMessage(text, attachmentUrls, opts) {
|
|
219
|
+
const inlineExtracted = opts?.inlineExtractedContent !== false;
|
|
219
220
|
let composed = text;
|
|
220
221
|
let composedForLlm = composed;
|
|
221
222
|
if (attachmentUrls.length > 0) {
|
|
@@ -229,7 +230,7 @@ ${lines.join("\n")}`;
|
|
|
229
230
|
let extractContent;
|
|
230
231
|
let fileUrls;
|
|
231
232
|
if (attachmentUrls.length > 0) {
|
|
232
|
-
const extractFiles = attachmentUrls.filter((u) => isServerExtractable(u.name));
|
|
233
|
+
const extractFiles = inlineExtracted ? attachmentUrls.filter((u) => isServerExtractable(u.name)) : [];
|
|
233
234
|
if (extractFiles.length > 0) {
|
|
234
235
|
const directives = [];
|
|
235
236
|
const sections = extractFiles.map((u) => {
|
|
@@ -290,7 +291,7 @@ Never assert absence from a partial read. Do not say "there is no X", "none", "n
|
|
|
290
291
|
Embedded values: a search term is often stored inside a larger string. A merchant "GODADDY" appears as "DNH*GODADDY#4070277042", and a card as "4140****2941". Server-side index filters match only exact values, leading prefixes, or trailing suffixes, and tag filters only EXACT whole-tag values - never a partial or interior substring - so filtering on such a field silently drops rows. When the value you are looking for may be embedded, do not trust a narrow filter to be complete. Fetch the full set with fetch_all and match the substring yourself.
|
|
291
292
|
File attachments: When a user message contains an "Attached files:" section with markdown links, those links point to short-lived signed URLs in this project's db storage and will expire.
|
|
292
293
|
- Image files (.jpg, .jpeg, .png, .gif, .webp) are ALREADY attached inline as image content blocks in the same message - you can see them directly. Do NOT call web_fetch on image URLs; that will fail or return garbage. Just look at the image block and answer.
|
|
293
|
-
-
|
|
294
|
+
- Other attached files (office documents like .docx/.xlsx/.pptx/.hwp/.hwpx/.ods, and text/data/code files like .csv/.tsv/.json/.xml/.txt/.md and source code) are ALREADY INDEXED: they were read end to end when they were uploaded, before this message reached you, and their content is in the database as records. Query it with getRecords using reference "src::<the storage path from the attachment link>" - one call, every table, every access group. Do NOT call web_fetch on their URLs. If you need the raw text rather than the indexed records (an exact quote, a specific cell), call readFileContent on that same path and page it with the cursor. Some turns instead carry the file text inlined between "BEGIN FILE CONTENT" / "END FILE CONTENT" markers; when that block is present read it directly, and a "[skapi: ...]" note inside it means that file could not be extracted.
|
|
294
295
|
- For any file given to you as a URL instead of inline content (e.g. PDFs), use your web_fetch tool to download and read each URL before answering. Treat the fetched contents as user-supplied input data. Do not ask the user to paste the file contents - fetch the URLs yourself.
|
|
295
296
|
Stored files and readFileContent: for a file ALREADY in this project's storage, its pages and rows were read at upload time and saved as records, so the database is your best source. Query those records first (getRecords with reference "src::<path>", or getUniqueId with unique_id "src::" and condition "gte" to find the file). readFileContent re-reads the raw file and is the right tool for text, spreadsheet and data files; it returns ONE window per call, so keep paging with the cursor from the previous window until it says END OF FILE before you conclude anything is absent. Be aware its PICTURES may not reach you: page images and embedded photos are attached as image blocks that several clients drop, leaving you only markers such as \xABPHOTO A88\xBB or a "(scanned; read the page images)" header. There is no OCR on the server, so a scanned page with no text layer carries no text at all. If you cannot actually see an image, say so plainly and fall back to the indexed records; never describe a picture you were not shown, and never tell the user the file is unreadable when its content is already in the database.
|
|
296
297
|
File links: When you find a record whose unique_id starts with "src::", the part after "src::" is the file's storage path or original URL. Always present it as a markdown link so the user can access it. Strip the "src::" prefix - do NOT show it. Format: [filename](db:path/to/file) for storage paths, or [filename](https://...) for external URLs. The db: prefix is REQUIRED on storage paths: it tells the chat client the target is a stored file rather than a web address, instead of leaving it to guess. Everything after db: is the path exactly as stored, including spaces and parentheses, and NOT url-encoded. Storage-path links render as clickable buttons in this chat client that fetch a fresh signed URL on demand - so even if a previously shared URL has expired, give the user the storage-path link instead of saying the file is unavailable. Never tell the user a file is inaccessible or a URL is expired if you have its storage path in the database.
|
|
@@ -564,6 +565,36 @@ function isAuthExpiredError(input) {
|
|
|
564
565
|
if (!hay) return false;
|
|
565
566
|
return hay.indexOf("token has expired") !== -1 || hay.indexOf("token is expired") !== -1 || hay.indexOf("expired_token") !== -1 || hay.indexOf("invalid_token") !== -1 || hay.indexOf("unauthorized") !== -1 || hay.indexOf("not authorized") !== -1 || hay.indexOf("invalid_request") !== -1 && hay.indexOf("token") !== -1;
|
|
566
567
|
}
|
|
568
|
+
function isProviderApiKeyError(input) {
|
|
569
|
+
if (!input) return false;
|
|
570
|
+
var blobs = [];
|
|
571
|
+
var push = function(v) {
|
|
572
|
+
if (typeof v === "string" && v) blobs.push(v);
|
|
573
|
+
};
|
|
574
|
+
if (typeof input === "string") push(input);
|
|
575
|
+
else {
|
|
576
|
+
push(input.message);
|
|
577
|
+
push(input.code);
|
|
578
|
+
push(input.type);
|
|
579
|
+
if (input.error) {
|
|
580
|
+
push(input.error.message);
|
|
581
|
+
push(input.error.code);
|
|
582
|
+
push(input.error.type);
|
|
583
|
+
}
|
|
584
|
+
if (input.body) {
|
|
585
|
+
push(input.body.message);
|
|
586
|
+
push(input.body.type);
|
|
587
|
+
if (input.body.error) {
|
|
588
|
+
push(input.body.error.message);
|
|
589
|
+
push(input.body.error.code);
|
|
590
|
+
push(input.body.error.type);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
var hay = blobs.join(" | ").toLowerCase();
|
|
595
|
+
if (!hay) return false;
|
|
596
|
+
return hay.indexOf("authentication_error") !== -1 || hay.indexOf("invalid_api_key") !== -1 || hay.indexOf("invalid x-api-key") !== -1 || hay.indexOf("incorrect api key") !== -1 || hay.indexOf("invalid api key") !== -1 || hay.indexOf("no api key provided") !== -1;
|
|
597
|
+
}
|
|
567
598
|
|
|
568
599
|
// src/engine/links.ts
|
|
569
600
|
var EXPIRED_ATTACHMENT_URL_HOST = "_expired_.url";
|
|
@@ -838,33 +869,85 @@ function truncateLabelForDisplay(label) {
|
|
|
838
869
|
// src/engine/budget.ts
|
|
839
870
|
var CONTEXT_WINDOW_DEFAULT = { claude: 2e5, openai: 128e3 };
|
|
840
871
|
var CONTEXT_WINDOW_BY_MODEL = {
|
|
841
|
-
// exact ids
|
|
872
|
+
// claude, exact ids
|
|
873
|
+
"claude-fable-5": 1e6,
|
|
842
874
|
"claude-opus-5": 1e6,
|
|
843
875
|
"claude-opus-4-8": 1e6,
|
|
844
876
|
"claude-opus-4-7": 1e6,
|
|
877
|
+
"claude-opus-4-6": 1e6,
|
|
878
|
+
"claude-opus-4-5": 2e5,
|
|
845
879
|
"claude-sonnet-5": 1e6,
|
|
846
880
|
"claude-sonnet-4-6": 1e6,
|
|
881
|
+
"claude-sonnet-4-5": 1e6,
|
|
847
882
|
"claude-sonnet-4": 2e5,
|
|
848
883
|
"claude-haiku-4-5": 2e5,
|
|
849
|
-
"
|
|
850
|
-
|
|
884
|
+
"claude-3-5-sonnet": 2e5,
|
|
885
|
+
// openai, exact ids
|
|
886
|
+
"gpt-5.6-sol": 105e4,
|
|
887
|
+
"gpt-5.6-terra": 105e4,
|
|
888
|
+
"gpt-5.6-luna": 105e4,
|
|
889
|
+
"gpt-5.5": 1e6,
|
|
890
|
+
"gpt-5.4": 105e4,
|
|
891
|
+
"gpt-5.4-mini": 4e5,
|
|
892
|
+
"gpt-5.4-nano": 4e5,
|
|
893
|
+
"gpt-4.1": 104e4,
|
|
894
|
+
"gpt-4o": 128e3,
|
|
895
|
+
"o1": 2e5,
|
|
896
|
+
"o1-pro": 2e5,
|
|
851
897
|
// family keys
|
|
898
|
+
"claude-fable": 1e6,
|
|
852
899
|
"claude-opus": 1e6,
|
|
853
900
|
"claude-sonnet": 1e6,
|
|
854
901
|
"claude-haiku": 2e5,
|
|
902
|
+
"gpt-5.6": 105e4,
|
|
903
|
+
"gpt-5": 128e3
|
|
904
|
+
};
|
|
905
|
+
var MAX_OUTPUT_BY_MODEL = {
|
|
906
|
+
// claude
|
|
907
|
+
"claude-fable-5": 128e3,
|
|
908
|
+
"claude-opus-5": 128e3,
|
|
909
|
+
"claude-opus-4-8": 128e3,
|
|
910
|
+
"claude-sonnet-5": 128e3,
|
|
911
|
+
"claude-sonnet-4-6": 64e3,
|
|
912
|
+
"claude-haiku-4-5": 64e3,
|
|
913
|
+
"claude-3-5-sonnet": 8e3,
|
|
914
|
+
// openai
|
|
915
|
+
"gpt-5.6-sol": 128e3,
|
|
916
|
+
"gpt-5.6-terra": 128e3,
|
|
917
|
+
"gpt-5.6-luna": 128e3,
|
|
918
|
+
"gpt-5.5": 128e3,
|
|
919
|
+
"gpt-5.4": 128e3,
|
|
920
|
+
"gpt-5.4-mini": 128e3,
|
|
921
|
+
"gpt-5.4-nano": 128e3,
|
|
922
|
+
"gpt-4.1": 16e3,
|
|
923
|
+
"gpt-4o": 4e3,
|
|
924
|
+
"o1": 1e5,
|
|
925
|
+
"o1-pro": 1e5,
|
|
926
|
+
// family keys
|
|
927
|
+
"claude-fable": 128e3,
|
|
928
|
+
"claude-opus": 128e3,
|
|
929
|
+
"claude-sonnet": 64e3,
|
|
930
|
+
"claude-haiku": 64e3,
|
|
855
931
|
"gpt-5.6": 128e3,
|
|
856
932
|
"gpt-5": 128e3
|
|
857
933
|
};
|
|
934
|
+
var DEFAULT_CONTEXT_WINDOW = 88e4;
|
|
858
935
|
var apiReportedContextWindows = {};
|
|
936
|
+
var apiReportedMaxOutput = {};
|
|
859
937
|
function registerModelContextWindows(models) {
|
|
860
938
|
if (!Array.isArray(models)) return;
|
|
861
939
|
for (var i = 0; i < models.length; i++) {
|
|
862
940
|
var m = models[i];
|
|
863
941
|
var id = (m && m.id ? String(m.id) : "").trim().toLowerCase();
|
|
942
|
+
if (!id) continue;
|
|
864
943
|
var reported = m ? Number(m.max_input_tokens) : NaN;
|
|
865
|
-
if (
|
|
944
|
+
if (Number.isFinite(reported) && reported > 0) {
|
|
866
945
|
apiReportedContextWindows[id] = Math.floor(reported);
|
|
867
946
|
}
|
|
947
|
+
var out = m ? Number(m.max_tokens) : NaN;
|
|
948
|
+
if (Number.isFinite(out) && out > 0) {
|
|
949
|
+
apiReportedMaxOutput[id] = Math.floor(out);
|
|
950
|
+
}
|
|
868
951
|
}
|
|
869
952
|
}
|
|
870
953
|
var projectContextWindows = {};
|
|
@@ -879,13 +962,16 @@ function getProjectContextWindow(projectId) {
|
|
|
879
962
|
var key = (projectId || "").trim();
|
|
880
963
|
return key && projectContextWindows[key] ? projectContextWindows[key] : null;
|
|
881
964
|
}
|
|
882
|
-
var
|
|
965
|
+
var MAX_OUTPUT_TOKENS = 25e3;
|
|
966
|
+
var OUTPUT_TOKEN_RESERVE = MAX_OUTPUT_TOKENS;
|
|
883
967
|
var TOOL_AND_RESPONSE_BUFFER = 4e3;
|
|
884
968
|
var MIN_INPUT_TOKEN_BUDGET = 8e3;
|
|
885
|
-
var
|
|
969
|
+
var MIN_PER_REQUEST_INPUT_CAP = 28e3;
|
|
970
|
+
var CLAUDE_PER_REQUEST_INPUT_CAP = MIN_PER_REQUEST_INPUT_CAP;
|
|
886
971
|
var MAX_HISTORY_MESSAGES = 20;
|
|
887
972
|
var HISTORY_TOKEN_BUDGET = 8e3;
|
|
888
|
-
var
|
|
973
|
+
var INPUT_CAP_RATIO = 0.16;
|
|
974
|
+
var CLAUDE_INPUT_CAP_RATIO = INPUT_CAP_RATIO;
|
|
889
975
|
var HISTORY_BUDGET_RATIO = 0.08;
|
|
890
976
|
function estimateTextTokens(text) {
|
|
891
977
|
return Math.ceil((text || "").length / 3);
|
|
@@ -893,38 +979,61 @@ function estimateTextTokens(text) {
|
|
|
893
979
|
function estimateMessageTokens(msg) {
|
|
894
980
|
return estimateTextTokens(msg.content) + estimateTextTokens(msg.role) + 6;
|
|
895
981
|
}
|
|
982
|
+
function resolveByModelId(apiTable, staticTable, model) {
|
|
983
|
+
var normalized = (model || "").trim().toLowerCase();
|
|
984
|
+
if (!normalized) return 0;
|
|
985
|
+
if (apiTable[normalized]) return apiTable[normalized];
|
|
986
|
+
if (staticTable[normalized]) return staticTable[normalized];
|
|
987
|
+
var parts = normalized.split("-");
|
|
988
|
+
for (var end = parts.length - 1; end > 0; end--) {
|
|
989
|
+
var family = parts.slice(0, end).join("-");
|
|
990
|
+
if (staticTable[family]) return staticTable[family];
|
|
991
|
+
}
|
|
992
|
+
return 0;
|
|
993
|
+
}
|
|
994
|
+
function getModelContextWindow(platform, model) {
|
|
995
|
+
return resolveByModelId(apiReportedContextWindows, CONTEXT_WINDOW_BY_MODEL, model) || CONTEXT_WINDOW_DEFAULT[platform];
|
|
996
|
+
}
|
|
997
|
+
function getMaxOutputTokens(platform, model) {
|
|
998
|
+
var cap = resolveByModelId(apiReportedMaxOutput, MAX_OUTPUT_BY_MODEL, model);
|
|
999
|
+
return cap ? Math.min(MAX_OUTPUT_TOKENS, cap) : MAX_OUTPUT_TOKENS;
|
|
1000
|
+
}
|
|
896
1001
|
function getContextWindow(platform, model, projectId) {
|
|
1002
|
+
var ceiling = getModelContextWindow(platform, model);
|
|
897
1003
|
var override = projectId ? getProjectContextWindow(projectId) : null;
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
return
|
|
1004
|
+
return Math.min(override || DEFAULT_CONTEXT_WINDOW, ceiling);
|
|
1005
|
+
}
|
|
1006
|
+
function contextBasedBudgetFor(platform, model, projectId) {
|
|
1007
|
+
var contextWindow = getContextWindow(platform, model, projectId);
|
|
1008
|
+
return Math.max(
|
|
1009
|
+
MIN_INPUT_TOKEN_BUDGET,
|
|
1010
|
+
contextWindow - getMaxOutputTokens(platform, model) - TOOL_AND_RESPONSE_BUFFER
|
|
1011
|
+
);
|
|
1012
|
+
}
|
|
1013
|
+
function getInputTokenBudget(platform, model, projectId) {
|
|
1014
|
+
var contextBasedBudget = contextBasedBudgetFor(platform, model, projectId);
|
|
1015
|
+
return Math.min(
|
|
1016
|
+
contextBasedBudget,
|
|
1017
|
+
Math.max(MIN_PER_REQUEST_INPUT_CAP, Math.round(contextBasedBudget * INPUT_CAP_RATIO))
|
|
1018
|
+
);
|
|
910
1019
|
}
|
|
911
1020
|
function stripFileBlocksFromHistory(content) {
|
|
912
1021
|
if (!content) return content;
|
|
913
1022
|
return content.replace(/```([^\n`]+?\.[^\s.`]+)\n[\s\S]*?```/g, "[file previously attached: $1]");
|
|
914
1023
|
}
|
|
915
1024
|
function buildBoundedChatMessages(options) {
|
|
916
|
-
var
|
|
917
|
-
var
|
|
918
|
-
MIN_INPUT_TOKEN_BUDGET,
|
|
919
|
-
contextWindow - OUTPUT_TOKEN_RESERVE - TOOL_AND_RESPONSE_BUFFER
|
|
920
|
-
);
|
|
921
|
-
var scaled = !!(options.projectId && getProjectContextWindow(options.projectId));
|
|
922
|
-
var claudeInputCap = scaled ? Math.max(CLAUDE_PER_REQUEST_INPUT_CAP, Math.round(contextBasedBudget * CLAUDE_INPUT_CAP_RATIO)) : CLAUDE_PER_REQUEST_INPUT_CAP;
|
|
923
|
-
var availableInputBudget = options.platform === "claude" ? Math.min(contextBasedBudget, claudeInputCap) : contextBasedBudget;
|
|
1025
|
+
var contextBasedBudget = contextBasedBudgetFor(options.platform, options.model, options.projectId);
|
|
1026
|
+
var availableInputBudget = getInputTokenBudget(options.platform, options.model, options.projectId);
|
|
924
1027
|
var systemCost = estimateTextTokens(options.systemPrompt) + 12;
|
|
925
|
-
var historyAllowance =
|
|
1028
|
+
var historyAllowance = Math.max(
|
|
1029
|
+
HISTORY_TOKEN_BUDGET,
|
|
1030
|
+
Math.round(contextBasedBudget * HISTORY_BUDGET_RATIO)
|
|
1031
|
+
);
|
|
926
1032
|
var budgetForHistory = Math.max(1e3, Math.min(historyAllowance, availableInputBudget - systemCost));
|
|
927
|
-
var maxHistoryMessages =
|
|
1033
|
+
var maxHistoryMessages = Math.max(
|
|
1034
|
+
MAX_HISTORY_MESSAGES,
|
|
1035
|
+
Math.round(MAX_HISTORY_MESSAGES * (budgetForHistory / HISTORY_TOKEN_BUDGET))
|
|
1036
|
+
);
|
|
928
1037
|
var windowed = options.history.slice(-maxHistoryMessages);
|
|
929
1038
|
var latestIndex = windowed.length - 1;
|
|
930
1039
|
var trimmed = windowed.map(function(m, i2) {
|
|
@@ -1317,7 +1426,6 @@ var WEB_FETCH_MAX_USES = 40;
|
|
|
1317
1426
|
var WEB_FETCH_MAX_CONTENT_TOKENS = 2e5;
|
|
1318
1427
|
var OPENAI_RESPONSES_API_URL = "https://api.openai.com/v1/responses";
|
|
1319
1428
|
var OPENAI_MODELS_API_URL = "https://api.openai.com/v1/models";
|
|
1320
|
-
var MAX_TOKENS = 25e3;
|
|
1321
1429
|
var DEFAULT_OPENAI_IMAGE_DETAIL = "auto";
|
|
1322
1430
|
var OPENAI_WEB_SEARCH_EXTERNAL_WEB_ACCESS = true;
|
|
1323
1431
|
var MCP_NAME = "BunnyQuery";
|
|
@@ -1557,7 +1665,7 @@ async function callClaudeWithPublicMcp(prompt, service, owner, messages, system,
|
|
|
1557
1665
|
owner,
|
|
1558
1666
|
userId,
|
|
1559
1667
|
model: model || DEFAULT_CLAUDE_MODEL,
|
|
1560
|
-
maxTokens:
|
|
1668
|
+
maxTokens: getMaxOutputTokens("claude", model || DEFAULT_CLAUDE_MODEL),
|
|
1561
1669
|
system,
|
|
1562
1670
|
extractContent,
|
|
1563
1671
|
fileUrls,
|
|
@@ -1602,7 +1710,7 @@ async function callOpenAIWithPublicMcp(prompt, service, owner, messages, system,
|
|
|
1602
1710
|
},
|
|
1603
1711
|
data: {
|
|
1604
1712
|
model: resolvedModel,
|
|
1605
|
-
max_output_tokens:
|
|
1713
|
+
max_output_tokens: getMaxOutputTokens("openai", resolvedModel),
|
|
1606
1714
|
...extractContent && extractContent.length ? { _skapi_extract: extractContent } : {},
|
|
1607
1715
|
...fileUrls && fileUrls.length ? { _skapi_file_urls: fileUrls } : {},
|
|
1608
1716
|
input: responseInput,
|
|
@@ -1632,6 +1740,29 @@ async function notifyAgentContinueIndexing(info) {
|
|
|
1632
1740
|
async function notifyAgentSaveAttachment(info) {
|
|
1633
1741
|
const { platform, service, owner, attachment, parsedContent } = info;
|
|
1634
1742
|
const continuing = !!info.continueIndexing;
|
|
1743
|
+
if (!continuing) {
|
|
1744
|
+
upsertIndexRunRecordSafe(service, attachment.storagePath, {
|
|
1745
|
+
status: "working",
|
|
1746
|
+
filename: attachment.name,
|
|
1747
|
+
started: Date.now(),
|
|
1748
|
+
queue: bgIndexingQueueName(info.userId, service),
|
|
1749
|
+
platform
|
|
1750
|
+
});
|
|
1751
|
+
}
|
|
1752
|
+
const tapDispatchFailure = (p) => {
|
|
1753
|
+
if (continuing) return p;
|
|
1754
|
+
return p.then(
|
|
1755
|
+
(ack) => ack,
|
|
1756
|
+
(err) => {
|
|
1757
|
+
upsertIndexRunRecordSafe(service, attachment.storagePath, {
|
|
1758
|
+
status: "error",
|
|
1759
|
+
finished: Date.now(),
|
|
1760
|
+
error: err && (err.message || String(err)) || "The indexing request could not be enqueued."
|
|
1761
|
+
});
|
|
1762
|
+
throw err;
|
|
1763
|
+
}
|
|
1764
|
+
);
|
|
1765
|
+
};
|
|
1635
1766
|
const visionFile = !parsedContent && isImageVisionFile(attachment.name, attachment.mime);
|
|
1636
1767
|
const renderFrom = Math.max(0, info.renderFrom || 0);
|
|
1637
1768
|
const renderPlaceholder = visionFile ? makeRenderPlaceholder(attachment.storagePath) : void 0;
|
|
@@ -1691,6 +1822,7 @@ async function notifyAgentSaveAttachment(info) {
|
|
|
1691
1822
|
save_media: !continuing
|
|
1692
1823
|
}))
|
|
1693
1824
|
} : {};
|
|
1825
|
+
const skapiFileUrls = attachment.url && attachment.storagePath ? { _skapi_file_urls: [{ path: attachment.storagePath, url: attachment.url }] } : {};
|
|
1694
1826
|
const userMessage = visionFile && renderPlaceholder ? buildIndexingRenderMessage(attachment, renderPlaceholder, renderFrom) : windowedRead && windowPlaceholder ? buildIndexingWindowMessage(attachment, windowPlaceholder, false) : continuing ? buildIndexingContinueMessage(attachment) : buildIndexingUserMessage(
|
|
1695
1827
|
attachment,
|
|
1696
1828
|
parsedContent ? { inlineContent: parsedContent } : placeholder ? { inlineContentPlaceholder: placeholder } : pagedRead ? { pagedRead: true } : void 0
|
|
@@ -1706,7 +1838,7 @@ async function notifyAgentSaveAttachment(info) {
|
|
|
1706
1838
|
if (platform === "openai") {
|
|
1707
1839
|
const resolvedModel2 = info.model || DEFAULT_OPENAI_MODEL;
|
|
1708
1840
|
const imageDetail = getOpenAIImageDetail(resolvedModel2);
|
|
1709
|
-
return clientSecretRequest({
|
|
1841
|
+
return tapDispatchFailure(clientSecretRequest({
|
|
1710
1842
|
clientSecretName: "openai",
|
|
1711
1843
|
queue: bgIndexingQueueName(info.userId, service),
|
|
1712
1844
|
service,
|
|
@@ -1720,12 +1852,13 @@ async function notifyAgentSaveAttachment(info) {
|
|
|
1720
1852
|
},
|
|
1721
1853
|
data: {
|
|
1722
1854
|
model: resolvedModel2,
|
|
1723
|
-
max_output_tokens:
|
|
1855
|
+
max_output_tokens: getMaxOutputTokens("openai", resolvedModel2),
|
|
1724
1856
|
// Nano-only transcription knobs. Indexing only; see variantIndexingOptions.
|
|
1725
1857
|
...variantIndexingOptions(resolvedModel2),
|
|
1726
1858
|
...skapiExtract,
|
|
1727
1859
|
...skapiRender,
|
|
1728
1860
|
...skapiWindow,
|
|
1861
|
+
...skapiFileUrls,
|
|
1729
1862
|
input: [
|
|
1730
1863
|
{ role: "system", content: systemPrompt },
|
|
1731
1864
|
{
|
|
@@ -1749,10 +1882,10 @@ async function notifyAgentSaveAttachment(info) {
|
|
|
1749
1882
|
]
|
|
1750
1883
|
]
|
|
1751
1884
|
}
|
|
1752
|
-
});
|
|
1885
|
+
}));
|
|
1753
1886
|
}
|
|
1754
1887
|
const resolvedModel = info.model || DEFAULT_CLAUDE_MODEL;
|
|
1755
|
-
return clientSecretRequest({
|
|
1888
|
+
return tapDispatchFailure(clientSecretRequest({
|
|
1756
1889
|
clientSecretName: "claude",
|
|
1757
1890
|
queue: bgIndexingQueueName(info.userId, service),
|
|
1758
1891
|
service,
|
|
@@ -1768,10 +1901,11 @@ async function notifyAgentSaveAttachment(info) {
|
|
|
1768
1901
|
},
|
|
1769
1902
|
data: {
|
|
1770
1903
|
model: resolvedModel,
|
|
1771
|
-
max_tokens:
|
|
1904
|
+
max_tokens: getMaxOutputTokens("claude", resolvedModel),
|
|
1772
1905
|
...skapiExtract,
|
|
1773
1906
|
...skapiRender,
|
|
1774
1907
|
...skapiWindow,
|
|
1908
|
+
...skapiFileUrls,
|
|
1775
1909
|
system: [
|
|
1776
1910
|
{
|
|
1777
1911
|
type: "text",
|
|
@@ -1807,7 +1941,7 @@ async function notifyAgentSaveAttachment(info) {
|
|
|
1807
1941
|
}
|
|
1808
1942
|
]
|
|
1809
1943
|
}
|
|
1810
|
-
});
|
|
1944
|
+
}));
|
|
1811
1945
|
}
|
|
1812
1946
|
function extractClaudeText(response) {
|
|
1813
1947
|
if (!Array.isArray(response?.content)) {
|
|
@@ -1868,6 +2002,21 @@ async function listOpenAIModels(service, owner) {
|
|
|
1868
2002
|
});
|
|
1869
2003
|
}
|
|
1870
2004
|
var BG_INDEXING_QUEUE_SUFFIX = "-bg";
|
|
2005
|
+
function indexDoneUniqueId(storagePath) {
|
|
2006
|
+
return "done::" + storagePath;
|
|
2007
|
+
}
|
|
2008
|
+
function runIndexUniqueId(storagePath) {
|
|
2009
|
+
return "run::" + storagePath;
|
|
2010
|
+
}
|
|
2011
|
+
function upsertIndexRunRecordSafe(service, storagePath, patch) {
|
|
2012
|
+
if (!service || !storagePath) return;
|
|
2013
|
+
try {
|
|
2014
|
+
const hook = chatEngineConfig().upsertIndexRunRecord;
|
|
2015
|
+
if (typeof hook !== "function") return;
|
|
2016
|
+
hook({ service, storagePath, patch });
|
|
2017
|
+
} catch (e) {
|
|
2018
|
+
}
|
|
2019
|
+
}
|
|
1871
2020
|
function bgIndexingQueueName(userId, service) {
|
|
1872
2021
|
return (userId || service || "") + BG_INDEXING_QUEUE_SUFFIX;
|
|
1873
2022
|
}
|
|
@@ -1891,13 +2040,20 @@ async function getChatHistory(params, fetchOptions) {
|
|
|
1891
2040
|
},
|
|
1892
2041
|
{ service: params.service, owner: params.owner },
|
|
1893
2042
|
params.queue ? { queue: params.queue } : {},
|
|
1894
|
-
params.status ? { status: params.status } : {}
|
|
2043
|
+
params.status ? { status: params.status } : {},
|
|
2044
|
+
params.queue_exact ? { queue_exact: true } : {},
|
|
2045
|
+
params.compact ? { compact: true } : {},
|
|
2046
|
+
params.queue_exclude ? { queue_exclude: params.queue_exclude } : {}
|
|
1895
2047
|
);
|
|
1896
2048
|
return chatEngineConfig().clientSecretRequestHistory(
|
|
1897
2049
|
p,
|
|
1898
2050
|
Object.assign({ ascending: false, limit: CHAT_HISTORY_PAGE_LIMIT }, fetchOptions)
|
|
1899
2051
|
);
|
|
1900
2052
|
}
|
|
2053
|
+
function buildHistoryItemFullId(platform, service, itemId) {
|
|
2054
|
+
const url = platform === "claude" ? ANTHROPIC_MESSAGES_API_URL : OPENAI_RESPONSES_API_URL;
|
|
2055
|
+
return `[POST]${url.toLowerCase()}#${service}:${itemId}`;
|
|
2056
|
+
}
|
|
1901
2057
|
|
|
1902
2058
|
// src/engine/history.ts
|
|
1903
2059
|
function filterListByClearHorizon(list, clearedAt) {
|
|
@@ -1947,6 +2103,278 @@ function parseIndexingRequestText(userText) {
|
|
|
1947
2103
|
continued: userText.indexOf("CONTINUE indexing") === 0
|
|
1948
2104
|
};
|
|
1949
2105
|
}
|
|
2106
|
+
var LIVE_INDEX_PROBE_LIMIT = 20;
|
|
2107
|
+
var BG_PROBE_TTL_MS = 4e3;
|
|
2108
|
+
var bgProbeCache = {};
|
|
2109
|
+
var bgProbeInflight = {};
|
|
2110
|
+
function probeBgQueue(params, opts) {
|
|
2111
|
+
const key = [params.service, params.owner, params.platform, params.queue, params.status, params.limit].join("|");
|
|
2112
|
+
const maxAge = opts && typeof opts.maxAgeMs === "number" ? opts.maxAgeMs : 0;
|
|
2113
|
+
const cached = bgProbeCache[key];
|
|
2114
|
+
if (maxAge > 0 && cached && Date.now() - cached.at < maxAge) {
|
|
2115
|
+
return Promise.resolve(cached);
|
|
2116
|
+
}
|
|
2117
|
+
const inflight = bgProbeInflight[key];
|
|
2118
|
+
if (inflight) return inflight;
|
|
2119
|
+
const p = Promise.resolve(getChatHistory(
|
|
2120
|
+
{ service: params.service, owner: params.owner, platform: params.platform, queue: params.queue, status: params.status },
|
|
2121
|
+
{ limit: params.limit, fetchMore: false }
|
|
2122
|
+
)).then(function(result) {
|
|
2123
|
+
const entry = { result, at: Date.now() };
|
|
2124
|
+
bgProbeCache[key] = entry;
|
|
2125
|
+
return entry;
|
|
2126
|
+
});
|
|
2127
|
+
bgProbeInflight[key] = p;
|
|
2128
|
+
p.then(function() {
|
|
2129
|
+
delete bgProbeInflight[key];
|
|
2130
|
+
}, function() {
|
|
2131
|
+
delete bgProbeInflight[key];
|
|
2132
|
+
});
|
|
2133
|
+
return p;
|
|
2134
|
+
}
|
|
2135
|
+
async function fetchLiveIndexingKeys(params) {
|
|
2136
|
+
const queue = bgIndexingQueueName(params.userId, params.service);
|
|
2137
|
+
const base = { service: params.service, owner: params.owner, platform: params.platform, queue };
|
|
2138
|
+
const [pending, running] = await Promise.all([
|
|
2139
|
+
probeBgQueue({ ...base, status: "pending", limit: LIVE_INDEX_PROBE_LIMIT }, { maxAgeMs: BG_PROBE_TTL_MS }),
|
|
2140
|
+
probeBgQueue({ ...base, status: "running", limit: LIVE_INDEX_PROBE_LIMIT }, { maxAgeMs: BG_PROBE_TTL_MS })
|
|
2141
|
+
]);
|
|
2142
|
+
const keys = /* @__PURE__ */ new Set();
|
|
2143
|
+
let truncated = false;
|
|
2144
|
+
for (const entry of [pending, running]) {
|
|
2145
|
+
const res = entry.result;
|
|
2146
|
+
const list = res && Array.isArray(res.list) ? res.list : [];
|
|
2147
|
+
if (list.length >= LIVE_INDEX_PROBE_LIMIT) truncated = true;
|
|
2148
|
+
for (const item of list) {
|
|
2149
|
+
const text = extractLastUserTextFromRequest(item && item.request_body);
|
|
2150
|
+
if (!text || !isIndexingRequestText(text)) continue;
|
|
2151
|
+
const ref = parseIndexingRequestText(text);
|
|
2152
|
+
if (!ref) continue;
|
|
2153
|
+
if (ref.path) keys.add(ref.path);
|
|
2154
|
+
if (ref.name) keys.add(ref.name);
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
return { keys, checked: !truncated, at: Math.min(pending.at, running.at) };
|
|
2158
|
+
}
|
|
2159
|
+
var BG_COVERAGE_MAX_PAGES = 2;
|
|
2160
|
+
var splitHistoryStates = {};
|
|
2161
|
+
var splitHistoryLocks = {};
|
|
2162
|
+
function freshSplitState() {
|
|
2163
|
+
return { bgBuffer: [], bgEnd: false, bgStarted: false, surfaceEnd: false, pendingSurface: null, surfaceCarry: [], lastSurfaceKeys: [], newestBgId: "" };
|
|
2164
|
+
}
|
|
2165
|
+
function noteBgIds(state, list) {
|
|
2166
|
+
for (const it of list) {
|
|
2167
|
+
const id = it && typeof it.id === "string" ? it.id : "";
|
|
2168
|
+
if (id && id > state.newestBgId) state.newestBgId = id;
|
|
2169
|
+
}
|
|
2170
|
+
}
|
|
2171
|
+
function __resetSplitHistoryState(key) {
|
|
2172
|
+
if (key !== void 0) {
|
|
2173
|
+
delete splitHistoryStates[key];
|
|
2174
|
+
delete splitHistoryLocks[key];
|
|
2175
|
+
return;
|
|
2176
|
+
}
|
|
2177
|
+
for (const k in splitHistoryStates) delete splitHistoryStates[k];
|
|
2178
|
+
for (const k in splitHistoryLocks) delete splitHistoryLocks[k];
|
|
2179
|
+
}
|
|
2180
|
+
var createdOf = (it) => {
|
|
2181
|
+
const c = Number(it && it.created);
|
|
2182
|
+
return isFinite(c) && c > 0 ? c : NaN;
|
|
2183
|
+
};
|
|
2184
|
+
var oldestCreated = (lst) => {
|
|
2185
|
+
let m = Infinity;
|
|
2186
|
+
for (const it of lst) {
|
|
2187
|
+
const c = createdOf(it);
|
|
2188
|
+
if (!isNaN(c) && c < m) m = c;
|
|
2189
|
+
}
|
|
2190
|
+
return m;
|
|
2191
|
+
};
|
|
2192
|
+
var SURFACE_EMPTY_MAX_PAGES = 10;
|
|
2193
|
+
async function getSplitChatHistory(params, fetchOptions, _fetchImpl) {
|
|
2194
|
+
const key = [params.service, params.owner, params.platform, params.userId || ""].join("|");
|
|
2195
|
+
const prev = splitHistoryLocks[key] || Promise.resolve();
|
|
2196
|
+
let releaseLock;
|
|
2197
|
+
const lockTail = new Promise((r) => {
|
|
2198
|
+
releaseLock = r;
|
|
2199
|
+
});
|
|
2200
|
+
const run = () => _getSplitChatHistoryLocked(key, params, fetchOptions, releaseLock, _fetchImpl);
|
|
2201
|
+
const p = prev.then(run, run);
|
|
2202
|
+
p.then((res) => {
|
|
2203
|
+
if (!res || !res.bgPending) releaseLock();
|
|
2204
|
+
}, () => releaseLock());
|
|
2205
|
+
splitHistoryLocks[key] = p.then(() => lockTail, () => lockTail);
|
|
2206
|
+
return p;
|
|
2207
|
+
}
|
|
2208
|
+
async function _getSplitChatHistoryLocked(key, params, fetchOptions, releaseLock, _fetchImpl) {
|
|
2209
|
+
const fetch = _fetchImpl || getChatHistory;
|
|
2210
|
+
const bgQueue = bgIndexingQueueName(params.userId, params.service);
|
|
2211
|
+
const base = { service: params.service, owner: params.owner, platform: params.platform };
|
|
2212
|
+
const fetchMore = !!(fetchOptions && fetchOptions.fetchMore);
|
|
2213
|
+
const limit = fetchOptions && fetchOptions.limit;
|
|
2214
|
+
const firstLoad = !splitHistoryStates[key];
|
|
2215
|
+
let headRefresh = false;
|
|
2216
|
+
if (!splitHistoryStates[key]) {
|
|
2217
|
+
splitHistoryStates[key] = freshSplitState();
|
|
2218
|
+
} else if (!fetchMore) {
|
|
2219
|
+
const prev = splitHistoryStates[key];
|
|
2220
|
+
if (prev.surfaceEnd && prev.bgEnd) {
|
|
2221
|
+
headRefresh = true;
|
|
2222
|
+
prev.pendingSurface = null;
|
|
2223
|
+
prev.surfaceCarry = [];
|
|
2224
|
+
prev.bgBuffer = [];
|
|
2225
|
+
} else {
|
|
2226
|
+
splitHistoryStates[key] = freshSplitState();
|
|
2227
|
+
}
|
|
2228
|
+
}
|
|
2229
|
+
const state = splitHistoryStates[key];
|
|
2230
|
+
if (state.pendingSurface && state.pendingSurface.forFetchMore !== fetchMore) {
|
|
2231
|
+
state.pendingSurface = null;
|
|
2232
|
+
}
|
|
2233
|
+
if (!state.pendingSurface) {
|
|
2234
|
+
if (state.surfaceEnd && !headRefresh) {
|
|
2235
|
+
state.pendingSurface = { list: [], endOfList: true, startKeyHistory: state.lastSurfaceKeys, forFetchMore: fetchMore };
|
|
2236
|
+
} else {
|
|
2237
|
+
const sOpts = { fetchMore };
|
|
2238
|
+
if (limit) sOpts.limit = limit;
|
|
2239
|
+
let s = await fetch({ ...base, queue_exclude: bgQueue }, sOpts);
|
|
2240
|
+
let hops = 0;
|
|
2241
|
+
while (s && !s.endOfList && !(s.list || []).length && hops < SURFACE_EMPTY_MAX_PAGES) {
|
|
2242
|
+
hops++;
|
|
2243
|
+
const nOpts = { fetchMore: true };
|
|
2244
|
+
if (limit) nOpts.limit = limit;
|
|
2245
|
+
s = await fetch({ ...base, queue_exclude: bgQueue }, nOpts);
|
|
2246
|
+
}
|
|
2247
|
+
state.pendingSurface = {
|
|
2248
|
+
list: s && Array.isArray(s.list) ? s.list : [],
|
|
2249
|
+
endOfList: !!(s && s.endOfList),
|
|
2250
|
+
startKeyHistory: s && Array.isArray(s.startKeyHistory) ? s.startKeyHistory : [],
|
|
2251
|
+
forFetchMore: fetchMore
|
|
2252
|
+
};
|
|
2253
|
+
}
|
|
2254
|
+
}
|
|
2255
|
+
const surface = state.pendingSurface;
|
|
2256
|
+
if (fetchOptions && fetchOptions.deferBg && (!state.bgEnd || headRefresh)) {
|
|
2257
|
+
const surfaceList0 = state.surfaceCarry.length ? state.surfaceCarry.concat(surface.list) : surface.list.slice();
|
|
2258
|
+
state.surfaceCarry = [];
|
|
2259
|
+
const emitNow = surfaceList0.concat(state.bgBuffer);
|
|
2260
|
+
state.bgBuffer = [];
|
|
2261
|
+
if (!headRefresh) state.surfaceEnd = surface.endOfList;
|
|
2262
|
+
state.lastSurfaceKeys = surface.startKeyHistory;
|
|
2263
|
+
state.pendingSurface = null;
|
|
2264
|
+
const bgPending = (async () => {
|
|
2265
|
+
try {
|
|
2266
|
+
const batch = [];
|
|
2267
|
+
if (headRefresh) {
|
|
2268
|
+
const bOpts = { fetchMore: false };
|
|
2269
|
+
if (limit) bOpts.limit = limit;
|
|
2270
|
+
const b = await fetch({ ...base, queue: bgQueue, queue_exact: true, compact: true }, bOpts);
|
|
2271
|
+
const bList = b && Array.isArray(b.list) ? b.list : [];
|
|
2272
|
+
for (const it of bList) {
|
|
2273
|
+
if (it && typeof it === "object") it._fromBgChain = true;
|
|
2274
|
+
batch.push(it);
|
|
2275
|
+
}
|
|
2276
|
+
const prevNewest = state.newestBgId;
|
|
2277
|
+
noteBgIds(state, bList);
|
|
2278
|
+
if (prevNewest && !(b && b.endOfList) && !bList.some((it) => it && it.id === prevNewest)) {
|
|
2279
|
+
state.bgEnd = false;
|
|
2280
|
+
state.bgStarted = true;
|
|
2281
|
+
}
|
|
2282
|
+
} else {
|
|
2283
|
+
let hops = 0;
|
|
2284
|
+
while (!state.bgEnd && hops < BG_COVERAGE_MAX_PAGES) {
|
|
2285
|
+
hops++;
|
|
2286
|
+
const bOpts = { fetchMore: state.bgStarted };
|
|
2287
|
+
if (limit) bOpts.limit = limit;
|
|
2288
|
+
const b = await fetch({ ...base, queue: bgQueue, queue_exact: true, compact: true }, bOpts);
|
|
2289
|
+
state.bgStarted = true;
|
|
2290
|
+
const bList = b && Array.isArray(b.list) ? b.list : [];
|
|
2291
|
+
for (const it of bList) {
|
|
2292
|
+
if (it && typeof it === "object") it._fromBgChain = true;
|
|
2293
|
+
batch.push(it);
|
|
2294
|
+
}
|
|
2295
|
+
noteBgIds(state, bList);
|
|
2296
|
+
state.bgEnd = !!(b && b.endOfList);
|
|
2297
|
+
if (!bList.length && !state.bgEnd) break;
|
|
2298
|
+
if (state.bgEnd) break;
|
|
2299
|
+
}
|
|
2300
|
+
}
|
|
2301
|
+
return { list: batch, endOfList: state.surfaceEnd && state.bgEnd };
|
|
2302
|
+
} finally {
|
|
2303
|
+
releaseLock();
|
|
2304
|
+
}
|
|
2305
|
+
})();
|
|
2306
|
+
return {
|
|
2307
|
+
list: emitNow,
|
|
2308
|
+
// A head-refreshed ended chain KNOWS it is still ended — reporting
|
|
2309
|
+
// the hardcoded false here was what un-gated the fill loop on every
|
|
2310
|
+
// tab return. Mid-walk it computes to false exactly as before (this
|
|
2311
|
+
// branch is only entered with bgEnd false then); the bg batch still
|
|
2312
|
+
// carries the final word for that case.
|
|
2313
|
+
endOfList: state.surfaceEnd && state.bgEnd,
|
|
2314
|
+
startKeyHistory: surface.startKeyHistory,
|
|
2315
|
+
firstLoad,
|
|
2316
|
+
bgPending
|
|
2317
|
+
};
|
|
2318
|
+
}
|
|
2319
|
+
const surfaceList = state.surfaceCarry.length ? state.surfaceCarry.concat(surface.list) : surface.list.slice();
|
|
2320
|
+
const boundary = surface.endOfList ? -Infinity : oldestCreated(surfaceList);
|
|
2321
|
+
if (headRefresh) {
|
|
2322
|
+
const hOpts = { fetchMore: false };
|
|
2323
|
+
if (limit) hOpts.limit = limit;
|
|
2324
|
+
const hb = await fetch({ ...base, queue: bgQueue, queue_exact: true, compact: true }, hOpts);
|
|
2325
|
+
const hbList = hb && Array.isArray(hb.list) ? hb.list : [];
|
|
2326
|
+
for (const it of hbList) {
|
|
2327
|
+
if (it && typeof it === "object") it._fromBgChain = true;
|
|
2328
|
+
state.bgBuffer.push(it);
|
|
2329
|
+
}
|
|
2330
|
+
const prevNewestH = state.newestBgId;
|
|
2331
|
+
noteBgIds(state, hbList);
|
|
2332
|
+
if (prevNewestH && !(hb && hb.endOfList) && !hbList.some((it) => it && it.id === prevNewestH)) {
|
|
2333
|
+
state.bgEnd = false;
|
|
2334
|
+
state.bgStarted = true;
|
|
2335
|
+
}
|
|
2336
|
+
} else if (boundary !== Infinity || surface.endOfList) {
|
|
2337
|
+
let hops = 0;
|
|
2338
|
+
while (!state.bgEnd && hops < BG_COVERAGE_MAX_PAGES) {
|
|
2339
|
+
const bufOldest = state.bgBuffer.length ? oldestCreated(state.bgBuffer) : Infinity;
|
|
2340
|
+
if (state.bgBuffer.length && bufOldest <= boundary) break;
|
|
2341
|
+
hops++;
|
|
2342
|
+
const bOpts = { fetchMore: state.bgStarted };
|
|
2343
|
+
if (limit) bOpts.limit = limit;
|
|
2344
|
+
const b = await fetch({ ...base, queue: bgQueue, queue_exact: true, compact: true }, bOpts);
|
|
2345
|
+
state.bgStarted = true;
|
|
2346
|
+
const bList = b && Array.isArray(b.list) ? b.list : [];
|
|
2347
|
+
for (const it of bList) {
|
|
2348
|
+
if (it && typeof it === "object") it._fromBgChain = true;
|
|
2349
|
+
state.bgBuffer.push(it);
|
|
2350
|
+
}
|
|
2351
|
+
noteBgIds(state, bList);
|
|
2352
|
+
state.bgEnd = !!(b && b.endOfList);
|
|
2353
|
+
if (!bList.length && !state.bgEnd) break;
|
|
2354
|
+
if (state.bgEnd) break;
|
|
2355
|
+
}
|
|
2356
|
+
}
|
|
2357
|
+
const emitSurface = surfaceList;
|
|
2358
|
+
state.surfaceCarry = [];
|
|
2359
|
+
const emitBg = state.bgBuffer;
|
|
2360
|
+
state.bgBuffer = [];
|
|
2361
|
+
const seen = {};
|
|
2362
|
+
for (const it of emitSurface) {
|
|
2363
|
+
if (it && typeof it.id === "string") seen[it.id] = true;
|
|
2364
|
+
}
|
|
2365
|
+
const merged = emitSurface.concat(emitBg.filter((it) => !(it && typeof it.id === "string" && seen[it.id])));
|
|
2366
|
+
if (!headRefresh) state.surfaceEnd = surface.endOfList;
|
|
2367
|
+
state.lastSurfaceKeys = surface.startKeyHistory;
|
|
2368
|
+
state.pendingSurface = null;
|
|
2369
|
+
return {
|
|
2370
|
+
list: merged,
|
|
2371
|
+
endOfList: state.surfaceEnd && state.bgEnd && state.bgBuffer.length === 0 && state.surfaceCarry.length === 0,
|
|
2372
|
+
// Bookkeeping only (both the consumers and the SDK treat it opaquely);
|
|
2373
|
+
// the real cursors are the SDK's internal ones plus this module's state.
|
|
2374
|
+
startKeyHistory: surface.startKeyHistory,
|
|
2375
|
+
firstLoad
|
|
2376
|
+
};
|
|
2377
|
+
}
|
|
1950
2378
|
function mapHistoryListToMessages(list, platform, opts) {
|
|
1951
2379
|
var mapped = [], runningItemIds = [];
|
|
1952
2380
|
var extractAssistantText = platform === "openai" ? extractOpenAIText : extractClaudeText;
|
|
@@ -1959,10 +2387,11 @@ function mapHistoryListToMessages(list, platform, opts) {
|
|
|
1959
2387
|
var isPending = isInProcess || isQueued;
|
|
1960
2388
|
var isFailed = item && item.status === "failed";
|
|
1961
2389
|
var response = isFailed ? item.error != null ? item.error : item.response_body : item && item.response_body != null ? item.response_body : item && item.error;
|
|
1962
|
-
var
|
|
1963
|
-
var
|
|
1964
|
-
var
|
|
1965
|
-
var
|
|
2390
|
+
var isCompact = !!(item && item.compact);
|
|
2391
|
+
var userText = isCompact ? typeof item.request_text === "string" ? item.request_text : "" : extractLastUserTextFromRequest(requestBody);
|
|
2392
|
+
var assistantText = isPending ? "" : isCompact ? (typeof item.response_text === "string" ? item.response_text : "").trim() : (extractAssistantText(response) || "").trim() || "";
|
|
2393
|
+
var isErrorResponse = !isPending && (isFailed || !isCompact && isErrorResponseBody(response));
|
|
2394
|
+
var reportedComplete = !!(item && item._isBgTask) && !isErrorResponse && (isCompact ? item.response_complete_marker === true : !!assistantText && assistantText.indexOf(INDEXING_COMPLETE_MARKER) !== -1);
|
|
1966
2395
|
if (reportedComplete) assistantText = assistantText.split(INDEXING_COMPLETE_MARKER).join("").trim();
|
|
1967
2396
|
var serverItemId = item && typeof item.id === "string" && item.id ? item.id : void 0;
|
|
1968
2397
|
var createdTs = Number(item && item.created);
|
|
@@ -1991,9 +2420,11 @@ function mapHistoryListToMessages(list, platform, opts) {
|
|
|
1991
2420
|
displayContent = sanitizeAttachmentLinksForHistory(userText, opts.projectId);
|
|
1992
2421
|
}
|
|
1993
2422
|
var userMsg = { role: "user", content: displayContent };
|
|
2423
|
+
if (item._fromBgChain) userMsg._fromBgChain = true;
|
|
1994
2424
|
if (isInProcess) userMsg.isPendingInProcess = true;
|
|
1995
2425
|
if (isQueued) userMsg.isPendingQueued = true;
|
|
1996
2426
|
if (isCancelledItem) userMsg.isCancelled = true;
|
|
2427
|
+
if (isCompact) userMsg._compact = true;
|
|
1997
2428
|
if (item._isBgTask) userMsg.isBackgroundTask = true;
|
|
1998
2429
|
if (indexFile) userMsg._indexFile = indexFile;
|
|
1999
2430
|
if (item._isOnBgQueue) userMsg._useBgQueue = true;
|
|
@@ -2003,6 +2434,8 @@ function mapHistoryListToMessages(list, platform, opts) {
|
|
|
2003
2434
|
}
|
|
2004
2435
|
if (isCancelledItem) ; else if (isInProcess) {
|
|
2005
2436
|
var ph = { role: "assistant", content: "", isPending: true, isPendingInProcess: true };
|
|
2437
|
+
if (userTs !== void 0) ph._ts = userTs;
|
|
2438
|
+
if (item._fromBgChain) ph._fromBgChain = true;
|
|
2006
2439
|
if (item._isBgTask) ph.isBackgroundTask = true;
|
|
2007
2440
|
if (serverItemId !== void 0) {
|
|
2008
2441
|
ph._serverItemId = serverItemId;
|
|
@@ -2011,19 +2444,26 @@ function mapHistoryListToMessages(list, platform, opts) {
|
|
|
2011
2444
|
mapped.push(ph);
|
|
2012
2445
|
} else if (isQueued) ; else if (isErrorResponse) {
|
|
2013
2446
|
var em = { role: "assistant", content: getErrorMessage(response), isError: true };
|
|
2447
|
+
if (item._fromBgChain) em._fromBgChain = true;
|
|
2014
2448
|
if (item._isBgTask) em.isBackgroundTask = true;
|
|
2015
2449
|
if (serverItemId !== void 0) em._serverItemId = serverItemId;
|
|
2016
2450
|
if (replyTs !== void 0) em._ts = replyTs;
|
|
2017
2451
|
mapped.push(em);
|
|
2018
2452
|
} else if (assistantText || reportedComplete) {
|
|
2019
2453
|
var okm = { role: "assistant", content: sanitizeAttachmentLinksForHistory(assistantText, opts.projectId, true) || EMPTY_INDEXING_REPLY };
|
|
2454
|
+
if (item._fromBgChain) okm._fromBgChain = true;
|
|
2020
2455
|
if (item._isBgTask) okm.isBackgroundTask = true;
|
|
2456
|
+
if (isCompact) okm._compact = true;
|
|
2021
2457
|
if (serverItemId !== void 0) okm._serverItemId = serverItemId;
|
|
2022
2458
|
if (replyTs !== void 0) okm._ts = replyTs;
|
|
2023
2459
|
if (reportedComplete) okm._indexComplete = true;
|
|
2024
2460
|
mapped.push(okm);
|
|
2025
2461
|
}
|
|
2026
2462
|
});
|
|
2463
|
+
if (opts.projectId) {
|
|
2464
|
+
var ownerKey = opts.projectId + "#" + platform;
|
|
2465
|
+
for (var oi = 0; oi < mapped.length; oi++) mapped[oi]._ownerKey = ownerKey;
|
|
2466
|
+
}
|
|
2027
2467
|
return { messages: mapped, runningItemIds };
|
|
2028
2468
|
}
|
|
2029
2469
|
|
|
@@ -2141,6 +2581,7 @@ var INDEXING_DRAIN_BUSY_POLL_MS = 8e3;
|
|
|
2141
2581
|
var INDEXING_DRAIN_CONFIRM_POLL_MS = 3e3;
|
|
2142
2582
|
var INDEXING_DRAIN_IDLE_LOOKS = 2;
|
|
2143
2583
|
var INDEXING_DRAIN_MIN_MS = 8e3;
|
|
2584
|
+
var _bgHistoryBatchSeq = 0;
|
|
2144
2585
|
var INDEXING_DRAIN_TIMEOUT_MS = 15 * 60 * 1e3;
|
|
2145
2586
|
var INDEXING_DRAIN_LOOK_TIMEOUT_MS = 45e3;
|
|
2146
2587
|
var INDEXING_DRAIN_NUDGE_MIN_GAP_MS = 1500;
|
|
@@ -2162,6 +2603,14 @@ function isPollStopped(res) {
|
|
|
2162
2603
|
}
|
|
2163
2604
|
var ChatSession = class {
|
|
2164
2605
|
constructor(host) {
|
|
2606
|
+
// ─── compact-stub hydration ─────────────────────────────────────────────
|
|
2607
|
+
// Split-fetch bg pages arrive as label stubs (no bodies). When the user
|
|
2608
|
+
// expands a row, the real reply text is fetched per item (csr-poll point
|
|
2609
|
+
// lookup) and MEMOIZED per chat: every later remap (first-page refresh,
|
|
2610
|
+
// queue-detect tick, cache restore) re-applies the memo, so a hydrated
|
|
2611
|
+
// bubble can never silently revert to its 200-char head.
|
|
2612
|
+
this._hydratedBodies = {};
|
|
2613
|
+
this._hydratingItems = {};
|
|
2165
2614
|
this.typewriterQueue = Promise.resolve();
|
|
2166
2615
|
/**
|
|
2167
2616
|
* Pick up indexing passes the WORKER minted, which no client ever dispatched.
|
|
@@ -2202,6 +2651,9 @@ var ChatSession = class {
|
|
|
2202
2651
|
typingAbort: false,
|
|
2203
2652
|
loadingHistory: false,
|
|
2204
2653
|
loadingOlderHistory: false,
|
|
2654
|
+
// A deferred bg stub batch (first-paint split) is still in flight; the
|
|
2655
|
+
// views show a small 'loading indexing history' hint while true.
|
|
2656
|
+
bgHistoryLoading: false,
|
|
2205
2657
|
historyEndOfList: false,
|
|
2206
2658
|
historyStartKeyHistory: [],
|
|
2207
2659
|
historyRequestToken: 0,
|
|
@@ -2351,10 +2803,12 @@ var ChatSession = class {
|
|
|
2351
2803
|
}
|
|
2352
2804
|
var queue = bgIndexingQueueName(id.userId, id.projectId);
|
|
2353
2805
|
var ask = function(status) {
|
|
2354
|
-
return Promise.resolve(
|
|
2355
|
-
{ service: id.projectId, owner: id.owner, platform, queue, status },
|
|
2356
|
-
{
|
|
2357
|
-
)).
|
|
2806
|
+
return Promise.resolve(probeBgQueue(
|
|
2807
|
+
{ service: id.projectId, owner: id.owner, platform, queue, status, limit: WORKER_PASS_ADOPT_LIMIT },
|
|
2808
|
+
{ maxAgeMs: BG_PROBE_TTL_MS }
|
|
2809
|
+
)).then(function(entry) {
|
|
2810
|
+
return entry.result;
|
|
2811
|
+
}).catch(function() {
|
|
2358
2812
|
return null;
|
|
2359
2813
|
});
|
|
2360
2814
|
};
|
|
@@ -2446,7 +2900,7 @@ var ChatSession = class {
|
|
|
2446
2900
|
* instead of merely unconfirmed.
|
|
2447
2901
|
*/
|
|
2448
2902
|
refreshLiveIndexState() {
|
|
2449
|
-
this._adoptWorkerIndexingPasses(0);
|
|
2903
|
+
this._adoptWorkerIndexingPasses(0, true);
|
|
2450
2904
|
}
|
|
2451
2905
|
/** Forget what we know about which files are indexing — but ONLY when the
|
|
2452
2906
|
* snapshot was taken for a different chat than the one on screen now. For a
|
|
@@ -2623,6 +3077,66 @@ var ChatSession = class {
|
|
|
2623
3077
|
if (!id.projectId || id.platform === "none") return "";
|
|
2624
3078
|
return id.projectId + "#" + id.platform;
|
|
2625
3079
|
}
|
|
3080
|
+
/** Re-apply memoized hydrated texts onto freshly-mapped messages. Both
|
|
3081
|
+
* clients call this right after their mapper runs (loadHistory does it
|
|
3082
|
+
* internally); it mutates the given array's items in place. */
|
|
3083
|
+
applyHydratedBodies(messages) {
|
|
3084
|
+
var key = this.getHistoryCacheKey();
|
|
3085
|
+
var memo = key ? this._hydratedBodies[key] : null;
|
|
3086
|
+
if (!memo) return;
|
|
3087
|
+
var id = this.host.getIdentity();
|
|
3088
|
+
for (var i = 0; i < messages.length; i++) {
|
|
3089
|
+
var m = messages[i];
|
|
3090
|
+
if (!m || !m._compact || m.role !== "assistant" || !m._serverItemId) continue;
|
|
3091
|
+
var text = memo[m._serverItemId];
|
|
3092
|
+
if (typeof text !== "string") continue;
|
|
3093
|
+
m.content = sanitizeAttachmentLinksForHistory(text, id.projectId, true) || EMPTY_INDEXING_REPLY;
|
|
3094
|
+
delete m._compact;
|
|
3095
|
+
}
|
|
3096
|
+
}
|
|
3097
|
+
/** Fetch the real response bodies for compact history stubs (one csr-poll
|
|
3098
|
+
* point lookup per item id), memoize, and swap them into the live list.
|
|
3099
|
+
* Best-effort: a failed lookup leaves the stub (its head + fallback line
|
|
3100
|
+
* still render) and a later expand retries. */
|
|
3101
|
+
hydrateCompactItems(itemIds) {
|
|
3102
|
+
var self = this;
|
|
3103
|
+
var lookup = chatEngineConfig().csrHistoryItemLookup;
|
|
3104
|
+
if (!lookup || !itemIds || !itemIds.length) return Promise.resolve();
|
|
3105
|
+
var id = this.host.getIdentity();
|
|
3106
|
+
var platform = id.platform;
|
|
3107
|
+
if (!id.projectId || platform !== "claude" && platform !== "openai") return Promise.resolve();
|
|
3108
|
+
var chatKey = this.getHistoryCacheKey();
|
|
3109
|
+
if (!chatKey) return Promise.resolve();
|
|
3110
|
+
var jobs = itemIds.map(function(itemId) {
|
|
3111
|
+
if (!itemId) return Promise.resolve();
|
|
3112
|
+
var already = self._hydratedBodies[chatKey] && self._hydratedBodies[chatKey][itemId] !== void 0;
|
|
3113
|
+
var inflightKey = chatKey + "|" + itemId;
|
|
3114
|
+
if (already || self._hydratingItems[inflightKey]) return Promise.resolve();
|
|
3115
|
+
self._hydratingItems[inflightKey] = true;
|
|
3116
|
+
return Promise.resolve(lookup(buildHistoryItemFullId(platform, id.projectId, itemId), id.projectId, id.owner)).then(function(body) {
|
|
3117
|
+
var text = ((platform === "openai" ? extractOpenAIText(body) : extractClaudeText(body)) || "").trim();
|
|
3118
|
+
if (text.indexOf(INDEXING_COMPLETE_MARKER) !== -1) text = text.split(INDEXING_COMPLETE_MARKER).join("").trim();
|
|
3119
|
+
if (!self._hydratedBodies[chatKey]) self._hydratedBodies[chatKey] = {};
|
|
3120
|
+
self._hydratedBodies[chatKey][itemId] = text;
|
|
3121
|
+
if (self.getHistoryCacheKey() !== chatKey) return;
|
|
3122
|
+
for (var i = 0; i < self.state.messages.length; i++) {
|
|
3123
|
+
var m = self.state.messages[i];
|
|
3124
|
+
if (m && m._compact && m.role === "assistant" && m._serverItemId === itemId) {
|
|
3125
|
+
m.content = sanitizeAttachmentLinksForHistory(text, id.projectId, true) || EMPTY_INDEXING_REPLY;
|
|
3126
|
+
delete m._compact;
|
|
3127
|
+
}
|
|
3128
|
+
}
|
|
3129
|
+
}).catch(function() {
|
|
3130
|
+
}).then(function() {
|
|
3131
|
+
delete self._hydratingItems[inflightKey];
|
|
3132
|
+
});
|
|
3133
|
+
});
|
|
3134
|
+
return Promise.all(jobs).then(function() {
|
|
3135
|
+
if (self.getHistoryCacheKey() !== chatKey) return;
|
|
3136
|
+
self.host.notify();
|
|
3137
|
+
self.updateHistoryCache();
|
|
3138
|
+
});
|
|
3139
|
+
}
|
|
2626
3140
|
updateHistoryCache() {
|
|
2627
3141
|
var key = this.getHistoryCacheKey();
|
|
2628
3142
|
if (!key) return;
|
|
@@ -2945,11 +3459,11 @@ var ChatSession = class {
|
|
|
2945
3459
|
bail = setTimeout(function() {
|
|
2946
3460
|
settle(null);
|
|
2947
3461
|
}, INDEXING_DRAIN_LOOK_TIMEOUT_MS);
|
|
2948
|
-
Promise.resolve(
|
|
2949
|
-
{ service: svcId, owner, platform, queue, status },
|
|
2950
|
-
{
|
|
2951
|
-
)).then(function(
|
|
2952
|
-
settle(
|
|
3462
|
+
Promise.resolve(probeBgQueue(
|
|
3463
|
+
{ service: svcId, owner, platform, queue, status, limit: WORKER_PASS_ADOPT_LIMIT },
|
|
3464
|
+
{ maxAgeMs: 0 }
|
|
3465
|
+
)).then(function(entry) {
|
|
3466
|
+
settle(entry.result);
|
|
2953
3467
|
}, function() {
|
|
2954
3468
|
settle(null);
|
|
2955
3469
|
});
|
|
@@ -3576,6 +4090,32 @@ var ChatSession = class {
|
|
|
3576
4090
|
if (e && e.id && self._indexKeyOf(e) === scoped) stoppedIds[e.id] = true;
|
|
3577
4091
|
});
|
|
3578
4092
|
this.state.stoppedIndexIds = stoppedIds;
|
|
4093
|
+
var runPath = group.path || "";
|
|
4094
|
+
if (!runPath) {
|
|
4095
|
+
(group.members || []).some(function(m) {
|
|
4096
|
+
var p = m && m.msg && m.msg._indexFile && m.msg._indexFile.path;
|
|
4097
|
+
if (p) {
|
|
4098
|
+
runPath = p;
|
|
4099
|
+
return true;
|
|
4100
|
+
}
|
|
4101
|
+
return false;
|
|
4102
|
+
});
|
|
4103
|
+
}
|
|
4104
|
+
if (!runPath) {
|
|
4105
|
+
this.bgTaskQueue.some(function(e) {
|
|
4106
|
+
if (e && e.storagePath && self._indexKeyOf(e) === scoped) {
|
|
4107
|
+
runPath = e.storagePath;
|
|
4108
|
+
return true;
|
|
4109
|
+
}
|
|
4110
|
+
return false;
|
|
4111
|
+
});
|
|
4112
|
+
}
|
|
4113
|
+
if (runPath) {
|
|
4114
|
+
var ident = this.host.getIdentity();
|
|
4115
|
+
if (ident && ident.projectId) {
|
|
4116
|
+
upsertIndexRunRecordSafe(ident.projectId, runPath, { status: "cancelled", finished: Date.now() });
|
|
4117
|
+
}
|
|
4118
|
+
}
|
|
3579
4119
|
}
|
|
3580
4120
|
this._adoptWorkerIndexingPasses(0);
|
|
3581
4121
|
var ids = group.cancellableIds || [];
|
|
@@ -4089,7 +4629,7 @@ var ChatSession = class {
|
|
|
4089
4629
|
if (isImageVisionFile(filename, mime)) return true;
|
|
4090
4630
|
return windowedIndexingEnabled() && isWindowedReadFile(filename, mime);
|
|
4091
4631
|
}
|
|
4092
|
-
_adoptWorkerIndexingPasses(attempt) {
|
|
4632
|
+
_adoptWorkerIndexingPasses(attempt, passive) {
|
|
4093
4633
|
var self = this;
|
|
4094
4634
|
if (this._adoptingWorkerPasses) return;
|
|
4095
4635
|
var id = this.host.getIdentity();
|
|
@@ -4099,10 +4639,12 @@ var ChatSession = class {
|
|
|
4099
4639
|
var svcId = id.projectId, owner = id.owner;
|
|
4100
4640
|
var queue = bgIndexingQueueName(id.userId, id.projectId);
|
|
4101
4641
|
var ask = function(status) {
|
|
4102
|
-
return Promise.resolve(
|
|
4103
|
-
{ service: svcId, owner, platform, queue, status },
|
|
4104
|
-
{
|
|
4105
|
-
)).
|
|
4642
|
+
return Promise.resolve(probeBgQueue(
|
|
4643
|
+
{ service: svcId, owner, platform, queue, status, limit: WORKER_PASS_ADOPT_LIMIT },
|
|
4644
|
+
{ maxAgeMs: 0 }
|
|
4645
|
+
)).then(function(entry) {
|
|
4646
|
+
return entry.result;
|
|
4647
|
+
}).catch(function() {
|
|
4106
4648
|
return null;
|
|
4107
4649
|
});
|
|
4108
4650
|
};
|
|
@@ -4124,6 +4666,7 @@ var ChatSession = class {
|
|
|
4124
4666
|
self.drainBgTaskQueue();
|
|
4125
4667
|
if (self._isTrackingAny(adoptedIds)) return;
|
|
4126
4668
|
}
|
|
4669
|
+
if (passive && !self._hasLiveIndexEvidence(svcId)) return;
|
|
4127
4670
|
if (attempt + 1 >= WORKER_PASS_ADOPT_ATTEMPTS.length) {
|
|
4128
4671
|
self._nudgeIndexingDrain();
|
|
4129
4672
|
return;
|
|
@@ -4132,12 +4675,30 @@ var ChatSession = class {
|
|
|
4132
4675
|
var later = self.host.getIdentity();
|
|
4133
4676
|
if (later.projectId !== svcId || later.platform !== platform) return;
|
|
4134
4677
|
if (self.isPollingPaused() || !self.host.isViewMounted()) return;
|
|
4135
|
-
self._adoptWorkerIndexingPasses(attempt + 1);
|
|
4678
|
+
self._adoptWorkerIndexingPasses(attempt + 1, passive);
|
|
4136
4679
|
}, WORKER_PASS_ADOPT_ATTEMPTS[attempt + 1]);
|
|
4137
4680
|
}, function() {
|
|
4138
4681
|
self._adoptingWorkerPasses = false;
|
|
4139
4682
|
});
|
|
4140
4683
|
}
|
|
4684
|
+
/** Anything at all suggesting THIS project's indexing may be live: a queued
|
|
4685
|
+
* local entry, a recorded live key (the adopt look just wrote them), or an
|
|
4686
|
+
* attached poll. Gates the passive adopt ladder's climb. */
|
|
4687
|
+
_hasLiveIndexEvidence(svcId) {
|
|
4688
|
+
for (var i = 0; i < this.bgTaskQueue.length; i++) {
|
|
4689
|
+
var e = this.bgTaskQueue[i];
|
|
4690
|
+
if (e && e.projectId === svcId) return true;
|
|
4691
|
+
}
|
|
4692
|
+
var keys = this.state.liveIndexKeys || {};
|
|
4693
|
+
for (var k in keys) {
|
|
4694
|
+
if (keys[k]) return true;
|
|
4695
|
+
}
|
|
4696
|
+
var found = false;
|
|
4697
|
+
this.historyItemPolls.forEach(function(h) {
|
|
4698
|
+
if (h && h.kind === "bg") found = true;
|
|
4699
|
+
});
|
|
4700
|
+
return found;
|
|
4701
|
+
}
|
|
4141
4702
|
/** Any of these ids still queued or still polled, i.e. surviving work. */
|
|
4142
4703
|
_isTrackingAny(ids) {
|
|
4143
4704
|
for (var i = 0; i < ids.length; i++) {
|
|
@@ -4228,7 +4789,10 @@ var ChatSession = class {
|
|
|
4228
4789
|
for (var i = this.bgTaskQueue.length - 1; i >= 0; i--) {
|
|
4229
4790
|
var e = this.bgTaskQueue[i];
|
|
4230
4791
|
if (e.projectId !== svcId || e.platform !== plat) continue;
|
|
4231
|
-
if (presentIds[e.id] && !pendingIds[e.id])
|
|
4792
|
+
if (presentIds[e.id] && !pendingIds[e.id]) {
|
|
4793
|
+
this._flipRunFromSettledEntry(e);
|
|
4794
|
+
this.bgTaskQueue.splice(i, 1);
|
|
4795
|
+
}
|
|
4232
4796
|
}
|
|
4233
4797
|
var bgPollBudget = MAX_CONCURRENT_BG_POLLS - this._countBgPolls();
|
|
4234
4798
|
var injectedAny = false;
|
|
@@ -4302,6 +4866,8 @@ var ChatSession = class {
|
|
|
4302
4866
|
self.host.notify();
|
|
4303
4867
|
self.updateHistoryCache();
|
|
4304
4868
|
if (!self._isWorkerDrivenIndexing(capturedEntry.filename, capturedEntry.mime)) {
|
|
4869
|
+
if (isNotExists) self._flipRunRecord(capturedEntry, "cancelled");
|
|
4870
|
+
else self._flipRunRecord(capturedEntry, "error", self._runErrorText(err));
|
|
4305
4871
|
self._nudgeIndexingDrain();
|
|
4306
4872
|
}
|
|
4307
4873
|
}).then(function() {
|
|
@@ -4333,6 +4899,74 @@ var ChatSession = class {
|
|
|
4333
4899
|
// memory (a reload or a closed tab ended it), and it stopped whenever the model claimed
|
|
4334
4900
|
// completion, which on an 88-page file happened at page 15. Continuing to dispatch here
|
|
4335
4901
|
// as well would now double-index every window.
|
|
4902
|
+
/** Fire the consumer's done::-marker hook for a run whose completion this
|
|
4903
|
+
* client knows DETERMINISTICALLY (see the two call sites in
|
|
4904
|
+
* maybeResumeIndexing). Best-effort by contract; identity-checked so a
|
|
4905
|
+
* project switch mid-settle cannot stamp the wrong service. */
|
|
4906
|
+
_mintDoneMarker(entry) {
|
|
4907
|
+
try {
|
|
4908
|
+
var mint = chatEngineConfig().mintIndexDoneMarker;
|
|
4909
|
+
if (!mint || !entry || !entry.storagePath || !entry.projectId) return;
|
|
4910
|
+
var id = this.host.getIdentity();
|
|
4911
|
+
if (!id || id.projectId !== entry.projectId) return;
|
|
4912
|
+
mint({ service: entry.projectId, storagePath: entry.storagePath });
|
|
4913
|
+
} catch (_e) {
|
|
4914
|
+
}
|
|
4915
|
+
}
|
|
4916
|
+
/** Short, storable form of an error body for the run:: record. */
|
|
4917
|
+
_runErrorText(response) {
|
|
4918
|
+
var msg = "";
|
|
4919
|
+
try {
|
|
4920
|
+
msg = String(getErrorMessage(response) || "");
|
|
4921
|
+
} catch (_e) {
|
|
4922
|
+
}
|
|
4923
|
+
msg = msg.replace(/\s+/g, " ").trim();
|
|
4924
|
+
return msg ? msg.slice(0, 300) : "Indexing failed.";
|
|
4925
|
+
}
|
|
4926
|
+
/** Close the records of a run whose pass settled OFF-POLL — the answer came
|
|
4927
|
+
* back as history (hidden tab, dead poll, resume refetch), so none of the
|
|
4928
|
+
* poll-side settle handlers ran. Only for SINGLE-PASS files, where one
|
|
4929
|
+
* settled pass is deterministically the whole run (the same contract as
|
|
4930
|
+
* maybeResumeIndexing's single-pass branch); paged files stay with their
|
|
4931
|
+
* drivers. Outcome is read from the settled bubbles' own flags, which is
|
|
4932
|
+
* all the history mapping left us. Best-effort and idempotent throughout. */
|
|
4933
|
+
_flipRunFromSettledEntry(entry) {
|
|
4934
|
+
try {
|
|
4935
|
+
if (!entry || !entry.storagePath || !entry.id || !entry.projectId) return;
|
|
4936
|
+
if (isPagedReadFile(entry.filename, entry.mime)) return;
|
|
4937
|
+
if (this.cancelledIndexKeys.has(this._indexKeyOf(entry))) return;
|
|
4938
|
+
if (this.state.stoppedIndexIds[entry.id]) return;
|
|
4939
|
+
var userMsg = null, replyMsg = null;
|
|
4940
|
+
this.state.messages.forEach(function(m) {
|
|
4941
|
+
if (m._serverItemId !== entry.id) return;
|
|
4942
|
+
if (m.role === "user") {
|
|
4943
|
+
if (!userMsg) userMsg = m;
|
|
4944
|
+
} else if (!replyMsg) replyMsg = m;
|
|
4945
|
+
});
|
|
4946
|
+
if (userMsg && userMsg.isCancelled || replyMsg && replyMsg.isCancelled) {
|
|
4947
|
+
this._flipRunRecord(entry, "cancelled");
|
|
4948
|
+
} else if (replyMsg && replyMsg.isError) {
|
|
4949
|
+
var errText = typeof replyMsg.content === "string" ? replyMsg.content.replace(/\s+/g, " ").trim().slice(0, 300) : "";
|
|
4950
|
+
this._flipRunRecord(entry, "error", errText || "Indexing failed.");
|
|
4951
|
+
} else if (replyMsg) {
|
|
4952
|
+
this._mintDoneMarker(entry);
|
|
4953
|
+
this._flipRunRecord(entry, "done");
|
|
4954
|
+
}
|
|
4955
|
+
} catch (_e) {
|
|
4956
|
+
}
|
|
4957
|
+
}
|
|
4958
|
+
/** Close the durable run:: record for an ending THIS client observed.
|
|
4959
|
+
* service comes from the ENTRY, not the current identity: unlike the done::
|
|
4960
|
+
* mint above, a status flip must land even if the user switched projects
|
|
4961
|
+
* mid-settle — otherwise the record lies 'working' forever. Best-effort
|
|
4962
|
+
* through upsertIndexRunRecordSafe; the consumer's precedence guard keeps
|
|
4963
|
+
* repeats and races harmless. */
|
|
4964
|
+
_flipRunRecord(entry, status, error) {
|
|
4965
|
+
if (!entry || !entry.storagePath || !entry.projectId) return;
|
|
4966
|
+
var patch = { status, finished: Date.now() };
|
|
4967
|
+
if (error) patch.error = error;
|
|
4968
|
+
upsertIndexRunRecordSafe(entry.projectId, entry.storagePath, patch);
|
|
4969
|
+
}
|
|
4336
4970
|
maybeResumeIndexing(entry, response, platform) {
|
|
4337
4971
|
var self = this;
|
|
4338
4972
|
var endOfClientChain = function() {
|
|
@@ -4342,27 +4976,43 @@ var ChatSession = class {
|
|
|
4342
4976
|
if (!entry || !entry.storagePath) return;
|
|
4343
4977
|
if (this.cancelledIndexKeys.has(this._indexKeyOf(entry))) return;
|
|
4344
4978
|
if (!isPagedReadFile(entry.filename, entry.mime)) {
|
|
4979
|
+
if (!isErrorResponseBody(response) && !this._isCancelledPollResult(response)) {
|
|
4980
|
+
this._mintDoneMarker(entry);
|
|
4981
|
+
this._flipRunRecord(entry, "done");
|
|
4982
|
+
} else if (this._isCancelledPollResult(response)) {
|
|
4983
|
+
this._flipRunRecord(entry, "cancelled");
|
|
4984
|
+
} else {
|
|
4985
|
+
this._flipRunRecord(entry, "error", this._runErrorText(response));
|
|
4986
|
+
}
|
|
4345
4987
|
endOfClientChain();
|
|
4346
4988
|
return;
|
|
4347
4989
|
}
|
|
4348
4990
|
if (isImageVisionFile(entry.filename, entry.mime)) return;
|
|
4349
4991
|
if (windowedIndexingEnabled() && isWindowedReadFile(entry.filename, entry.mime)) return;
|
|
4350
4992
|
if (isErrorResponseBody(response)) {
|
|
4993
|
+
this._flipRunRecord(entry, "error", this._runErrorText(response));
|
|
4351
4994
|
endOfClientChain();
|
|
4352
4995
|
return;
|
|
4353
4996
|
}
|
|
4354
4997
|
var answer = (platform === "openai" ? extractOpenAIText(response) : extractClaudeText(response)) || "";
|
|
4355
4998
|
if (answer.indexOf(INDEXING_COMPLETE_MARKER) !== -1) {
|
|
4999
|
+
this._mintDoneMarker(entry);
|
|
5000
|
+
this._flipRunRecord(entry, "done");
|
|
4356
5001
|
endOfClientChain();
|
|
4357
5002
|
return;
|
|
4358
5003
|
}
|
|
4359
5004
|
var pass = (entry.resumePass || 0) + 1;
|
|
4360
5005
|
if (pass > MAX_INDEXING_RESUME_PASSES) {
|
|
5006
|
+
this._flipRunRecord(entry, "error", "Stopped after " + MAX_INDEXING_RESUME_PASSES + " passes without finishing.");
|
|
4361
5007
|
endOfClientChain();
|
|
4362
5008
|
return;
|
|
4363
5009
|
}
|
|
4364
5010
|
var id = this.host.getIdentity();
|
|
4365
|
-
if (!id || id.platform === "none" || id.projectId !== entry.projectId)
|
|
5011
|
+
if (!id || id.platform === "none" || id.projectId !== entry.projectId) {
|
|
5012
|
+
this._flipRunRecord(entry, "error", "Indexing stopped: the session or project changed before the file finished.");
|
|
5013
|
+
endOfClientChain();
|
|
5014
|
+
return;
|
|
5015
|
+
}
|
|
4366
5016
|
this.trackIndexDispatch(notifyAgentContinueIndexing({
|
|
4367
5017
|
platform: id.platform,
|
|
4368
5018
|
model: id.model,
|
|
@@ -4439,8 +5089,9 @@ var ChatSession = class {
|
|
|
4439
5089
|
var projectId = id.projectId, owner = id.owner;
|
|
4440
5090
|
var options = { fetchMore };
|
|
4441
5091
|
if (fetchMore && this.state.historyStartKeyHistory.length) options.startKeyHistory = this.state.historyStartKeyHistory.slice();
|
|
5092
|
+
if (!fetchMore) options.deferBg = true;
|
|
4442
5093
|
var fetchHistory = function() {
|
|
4443
|
-
return
|
|
5094
|
+
return getSplitChatHistory({ service: projectId, owner, platform, userId: id.userId }, options);
|
|
4444
5095
|
};
|
|
4445
5096
|
return Promise.resolve().then(fetchHistory).catch(function(err) {
|
|
4446
5097
|
if (isAuthExpiredError(err) && !isNonRetryableRequestError(err)) return self.host.refreshSession().then(fetchHistory);
|
|
@@ -4450,7 +5101,8 @@ var ChatSession = class {
|
|
|
4450
5101
|
var chatList = history && Array.isArray(history.list) ? history.list : [];
|
|
4451
5102
|
chatList.forEach(function(item) {
|
|
4452
5103
|
if (isBgIndexingQueue(item.queue_name)) {
|
|
4453
|
-
|
|
5104
|
+
var clsText = item.compact ? item.request_text : extractLastUserTextFromRequest(item.request_body);
|
|
5105
|
+
if (isIndexingRequestText(clsText)) item._isBgTask = true;
|
|
4454
5106
|
else item._isOnBgQueue = true;
|
|
4455
5107
|
}
|
|
4456
5108
|
});
|
|
@@ -4463,15 +5115,55 @@ var ChatSession = class {
|
|
|
4463
5115
|
projectId: id.projectId,
|
|
4464
5116
|
formatIndexingLabel: self.host.formatIndexingLabel
|
|
4465
5117
|
}).messages;
|
|
5118
|
+
self.applyHydratedBodies(mapped);
|
|
4466
5119
|
var keptOlderPages = false;
|
|
5120
|
+
var keptScreenAwaitingBg = false;
|
|
4467
5121
|
if (fetchMore) {
|
|
4468
|
-
|
|
5122
|
+
var incomingKeys = {};
|
|
5123
|
+
mapped.forEach(function(m) {
|
|
5124
|
+
if (m._serverItemId) incomingKeys[m._serverItemId + "|" + m.role] = m;
|
|
5125
|
+
});
|
|
5126
|
+
var existing = self.state.messages.filter(function(m) {
|
|
5127
|
+
if (!m._serverItemId) return true;
|
|
5128
|
+
var inc = incomingKeys[m._serverItemId + "|" + m.role];
|
|
5129
|
+
if (!inc) return true;
|
|
5130
|
+
if (m._cancelling) inc._cancelling = m._cancelling;
|
|
5131
|
+
if (m._cancelError) inc._cancelError = m._cancelError;
|
|
5132
|
+
return false;
|
|
5133
|
+
});
|
|
5134
|
+
var mergedList = [];
|
|
5135
|
+
var pi = 0, ei = 0;
|
|
5136
|
+
while (pi < mapped.length && ei < existing.length) {
|
|
5137
|
+
var pm = mapped[pi], em = existing[ei];
|
|
5138
|
+
var eid = em._serverItemId;
|
|
5139
|
+
if (typeof eid !== "string") break;
|
|
5140
|
+
var pid = pm._serverItemId;
|
|
5141
|
+
if (typeof pid !== "string" || pid <= eid) {
|
|
5142
|
+
mergedList.push(pm);
|
|
5143
|
+
pi++;
|
|
5144
|
+
} else {
|
|
5145
|
+
mergedList.push(em);
|
|
5146
|
+
ei++;
|
|
5147
|
+
}
|
|
5148
|
+
}
|
|
5149
|
+
while (pi < mapped.length) mergedList.push(mapped[pi++]);
|
|
5150
|
+
while (ei < existing.length) mergedList.push(existing[ei++]);
|
|
5151
|
+
self.state.messages = mergedList;
|
|
5152
|
+
} else if (!mapped.length && history && (history.endOfList === false || history.bgPending) && self.state.messages.some(function(m) {
|
|
5153
|
+
return m._ownerKey === void 0 || m._ownerKey === loadKey;
|
|
5154
|
+
})) {
|
|
5155
|
+
if (history.endOfList !== false) keptScreenAwaitingBg = true;
|
|
4469
5156
|
} else {
|
|
4470
5157
|
if (self.state.typing) self.state.typingAbort = true;
|
|
4471
5158
|
var serverIds = {};
|
|
4472
5159
|
mapped.forEach(function(m) {
|
|
4473
5160
|
if (m._serverItemId) serverIds[m._serverItemId] = 1;
|
|
4474
5161
|
});
|
|
5162
|
+
var surfaceOldestId = void 0;
|
|
5163
|
+
mapped.forEach(function(m) {
|
|
5164
|
+
if (typeof m._serverItemId !== "string" || m._fromBgChain) return;
|
|
5165
|
+
if (surfaceOldestId === void 0 || m._serverItemId < surfaceOldestId) surfaceOldestId = m._serverItemId;
|
|
5166
|
+
});
|
|
4475
5167
|
var locallyCancelled = {};
|
|
4476
5168
|
self.state.messages.forEach(function(m) {
|
|
4477
5169
|
if (m.isCancelled && m._serverItemId) locallyCancelled[m._serverItemId] = m;
|
|
@@ -4512,13 +5204,45 @@ var ChatSession = class {
|
|
|
4512
5204
|
var sharesPage1 = self.state.messages.some(function(m) {
|
|
4513
5205
|
return typeof m._serverItemId === "string" && !!serverIds[m._serverItemId];
|
|
4514
5206
|
});
|
|
4515
|
-
var
|
|
5207
|
+
var deferredBg = !!(history && history.bgPending);
|
|
5208
|
+
var retainBoundary = surfaceOldestId !== void 0 ? surfaceOldestId : oldestInPage1;
|
|
5209
|
+
var retainedOlder = !sharesPage1 || retainBoundary === void 0 ? [] : self.state.messages.filter(function(m) {
|
|
4516
5210
|
if (typeof m._serverItemId !== "string") return false;
|
|
4517
5211
|
if (m._ownerKey !== void 0 && m._ownerKey !== loadKey) return false;
|
|
4518
|
-
|
|
5212
|
+
if (deferredBg && m.isBackgroundTask) return true;
|
|
5213
|
+
if (m._fromBgChain) return true;
|
|
5214
|
+
return m._serverItemId < retainBoundary;
|
|
4519
5215
|
});
|
|
4520
|
-
|
|
4521
|
-
|
|
5216
|
+
var prependOlder = [];
|
|
5217
|
+
var interleave = [];
|
|
5218
|
+
retainedOlder.forEach(function(m) {
|
|
5219
|
+
var sid = m._serverItemId;
|
|
5220
|
+
if (serverIds[sid]) return;
|
|
5221
|
+
if (retainBoundary !== void 0 && sid < retainBoundary) prependOlder.push(m);
|
|
5222
|
+
else interleave.push(m);
|
|
5223
|
+
});
|
|
5224
|
+
var page1 = mapped;
|
|
5225
|
+
if (interleave.length) {
|
|
5226
|
+
var mergedP = [];
|
|
5227
|
+
var ii2 = 0, mi2 = 0;
|
|
5228
|
+
while (ii2 < interleave.length && mi2 < mapped.length) {
|
|
5229
|
+
var iv = interleave[ii2], mv = mapped[mi2];
|
|
5230
|
+
var mid2 = typeof mv._serverItemId === "string" ? mv._serverItemId : void 0;
|
|
5231
|
+
if (mid2 === void 0) break;
|
|
5232
|
+
if (iv._serverItemId <= mid2) {
|
|
5233
|
+
mergedP.push(iv);
|
|
5234
|
+
ii2++;
|
|
5235
|
+
} else {
|
|
5236
|
+
mergedP.push(mv);
|
|
5237
|
+
mi2++;
|
|
5238
|
+
}
|
|
5239
|
+
}
|
|
5240
|
+
while (ii2 < interleave.length) mergedP.push(interleave[ii2++]);
|
|
5241
|
+
while (mi2 < mapped.length) mergedP.push(mapped[mi2++]);
|
|
5242
|
+
page1 = mergedP;
|
|
5243
|
+
}
|
|
5244
|
+
keptOlderPages = prependOlder.length > 0 || interleave.length > 0;
|
|
5245
|
+
self.state.messages = prependOlder.length ? prependOlder.concat(page1) : page1;
|
|
4522
5246
|
rescued.forEach(function(m) {
|
|
4523
5247
|
self.state.messages.push(m);
|
|
4524
5248
|
});
|
|
@@ -4554,9 +5278,14 @@ var ChatSession = class {
|
|
|
4554
5278
|
self.state.historyEndOfList = !!(history && history.endOfList);
|
|
4555
5279
|
self.state.historyStartKeyHistory = history && Array.isArray(history.startKeyHistory) ? history.startKeyHistory : [];
|
|
4556
5280
|
var clearedAt = self.host.getClearedAt();
|
|
4557
|
-
if (clearedAt
|
|
4558
|
-
var
|
|
4559
|
-
|
|
5281
|
+
if (clearedAt) {
|
|
5282
|
+
var surfaceItems = chatList.filter(function(it) {
|
|
5283
|
+
return !(it && it._fromBgChain);
|
|
5284
|
+
});
|
|
5285
|
+
if (surfaceItems.length > 0) {
|
|
5286
|
+
var oldestUpdated = Number(surfaceItems[surfaceItems.length - 1] && surfaceItems[surfaceItems.length - 1].updated);
|
|
5287
|
+
if (isFinite(oldestUpdated) && oldestUpdated <= clearedAt) self.state.historyEndOfList = true;
|
|
5288
|
+
}
|
|
4560
5289
|
}
|
|
4561
5290
|
}
|
|
4562
5291
|
if (self.state.historyRequestToken === token) {
|
|
@@ -4565,6 +5294,85 @@ var ChatSession = class {
|
|
|
4565
5294
|
}
|
|
4566
5295
|
self.updateHistoryCache();
|
|
4567
5296
|
self.host.notify();
|
|
5297
|
+
var bgPending = !fetchMore && history && history.bgPending;
|
|
5298
|
+
if (bgPending) {
|
|
5299
|
+
var batchId = ++_bgHistoryBatchSeq;
|
|
5300
|
+
if (history.endOfList !== true && history.firstLoad === true) {
|
|
5301
|
+
self.state.bgHistoryLoading = true;
|
|
5302
|
+
self.host.notify();
|
|
5303
|
+
}
|
|
5304
|
+
var releaseBgFlag = function() {
|
|
5305
|
+
if (_bgHistoryBatchSeq === batchId) self.state.bgHistoryLoading = false;
|
|
5306
|
+
};
|
|
5307
|
+
bgPending.then(function(batch) {
|
|
5308
|
+
if (token !== self.state.gateRefreshToken) {
|
|
5309
|
+
releaseBgFlag();
|
|
5310
|
+
return;
|
|
5311
|
+
}
|
|
5312
|
+
var bList = batch && Array.isArray(batch.list) ? batch.list : [];
|
|
5313
|
+
bList.forEach(function(item) {
|
|
5314
|
+
if (isBgIndexingQueue(item.queue_name)) {
|
|
5315
|
+
var t = item.compact ? item.request_text : extractLastUserTextFromRequest(item.request_body);
|
|
5316
|
+
if (isIndexingRequestText(t)) item._isBgTask = true;
|
|
5317
|
+
else item._isOnBgQueue = true;
|
|
5318
|
+
}
|
|
5319
|
+
});
|
|
5320
|
+
var sorted = bList.sort(function(a, b) {
|
|
5321
|
+
var ai = typeof a.id === "string" ? a.id : "", bi = typeof b.id === "string" ? b.id : "";
|
|
5322
|
+
return ai > bi ? -1 : ai < bi ? 1 : 0;
|
|
5323
|
+
});
|
|
5324
|
+
var m2 = mapHistoryListToMessages(sorted, platform, {
|
|
5325
|
+
clearedAt: self.host.getClearedAt(),
|
|
5326
|
+
projectId: id.projectId,
|
|
5327
|
+
formatIndexingLabel: self.host.formatIndexingLabel
|
|
5328
|
+
}).messages;
|
|
5329
|
+
self.applyHydratedBodies(m2);
|
|
5330
|
+
if (keptScreenAwaitingBg && !m2.length && batch && batch.endOfList === true) {
|
|
5331
|
+
self.state.messages = self.state.messages.filter(function(m) {
|
|
5332
|
+
if (typeof m._serverItemId !== "string") return true;
|
|
5333
|
+
if (m._ownerKey !== void 0 && m._ownerKey !== loadKey) return true;
|
|
5334
|
+
return false;
|
|
5335
|
+
});
|
|
5336
|
+
self.state.historyEndOfList = true;
|
|
5337
|
+
releaseBgFlag();
|
|
5338
|
+
self.updateHistoryCache();
|
|
5339
|
+
self.host.notify();
|
|
5340
|
+
return;
|
|
5341
|
+
}
|
|
5342
|
+
var incoming = {};
|
|
5343
|
+
m2.forEach(function(m) {
|
|
5344
|
+
if (m._serverItemId) incoming[m._serverItemId + "|" + m.role] = true;
|
|
5345
|
+
});
|
|
5346
|
+
var baseList = self.state.messages.filter(function(m) {
|
|
5347
|
+
return !(m._serverItemId && incoming[m._serverItemId + "|" + m.role]);
|
|
5348
|
+
});
|
|
5349
|
+
var mergedList2 = [];
|
|
5350
|
+
var pi2 = 0, ei2 = 0;
|
|
5351
|
+
while (pi2 < m2.length && ei2 < baseList.length) {
|
|
5352
|
+
var pm2 = m2[pi2], em2 = baseList[ei2];
|
|
5353
|
+
var eid2 = em2._serverItemId;
|
|
5354
|
+
if (typeof eid2 !== "string") break;
|
|
5355
|
+
var pid2 = pm2._serverItemId;
|
|
5356
|
+
if (typeof pid2 !== "string" || pid2 <= eid2) {
|
|
5357
|
+
mergedList2.push(pm2);
|
|
5358
|
+
pi2++;
|
|
5359
|
+
} else {
|
|
5360
|
+
mergedList2.push(em2);
|
|
5361
|
+
ei2++;
|
|
5362
|
+
}
|
|
5363
|
+
}
|
|
5364
|
+
while (pi2 < m2.length) mergedList2.push(m2[pi2++]);
|
|
5365
|
+
while (ei2 < baseList.length) mergedList2.push(baseList[ei2++]);
|
|
5366
|
+
self.state.messages = mergedList2;
|
|
5367
|
+
if (batch && batch.endOfList === true) self.state.historyEndOfList = true;
|
|
5368
|
+
releaseBgFlag();
|
|
5369
|
+
self.updateHistoryCache();
|
|
5370
|
+
self.host.notify();
|
|
5371
|
+
}, function() {
|
|
5372
|
+
releaseBgFlag();
|
|
5373
|
+
self.host.notify();
|
|
5374
|
+
});
|
|
5375
|
+
}
|
|
4568
5376
|
if (!fetchMore) {
|
|
4569
5377
|
var bgAllow = {};
|
|
4570
5378
|
var bgHistBudget = MAX_CONCURRENT_BG_POLLS - self._countBgPolls();
|
|
@@ -4661,7 +5469,7 @@ var ChatSession = class {
|
|
|
4661
5469
|
var self = this;
|
|
4662
5470
|
var id = this.host.getIdentity();
|
|
4663
5471
|
att.status = "uploading";
|
|
4664
|
-
att.progress =
|
|
5472
|
+
att.progress = null;
|
|
4665
5473
|
att.errorMessage = "";
|
|
4666
5474
|
att.errorCode = "";
|
|
4667
5475
|
att.errorDetail = "";
|
|
@@ -4892,6 +5700,7 @@ var ChatSession = class {
|
|
|
4892
5700
|
};
|
|
4893
5701
|
|
|
4894
5702
|
// src/engine/indexing_groups.ts
|
|
5703
|
+
var RUN_RECORD_WORKING_STALE_MS = 6 * 60 * 60 * 1e3;
|
|
4895
5704
|
var INDEXING_LABEL_RE = /^(Re)?[Ii]ndexing(\s*\(continuing\))?\s*:?\s+(.+)$/;
|
|
4896
5705
|
var LEADING_MD_LINK_RE = /^\[([^\]]+)\]\(([^)]+)\)/;
|
|
4897
5706
|
function parseIndexingLabel(content) {
|
|
@@ -4946,10 +5755,12 @@ function buildChatDisplayList(messages, opts) {
|
|
|
4946
5755
|
var list = Array.isArray(messages) ? messages : [];
|
|
4947
5756
|
var liveIndexKeys = opts && opts.liveIndexKeys || {};
|
|
4948
5757
|
var liveIndexChecked = !!(opts && opts.liveIndexChecked);
|
|
5758
|
+
var doneKeys = opts && opts.doneKeys || {};
|
|
4949
5759
|
var stoppedIndexIds = opts && opts.stoppedIndexIds || {};
|
|
4950
5760
|
var windowedIndexing = opts && opts.windowedIndexing !== void 0 ? !!opts.windowedIndexing : windowedIndexingEnabled();
|
|
4951
5761
|
var hasMoreHistory = !!(opts && opts.hasMoreHistory);
|
|
4952
5762
|
var loadingOlderHistory = !!(opts && opts.loadingOlderHistory);
|
|
5763
|
+
var stubPlatform = opts && opts.stubPlatform;
|
|
4953
5764
|
var groups = {};
|
|
4954
5765
|
var order = [];
|
|
4955
5766
|
var runOfIndex = new Array(list.length);
|
|
@@ -5122,11 +5933,11 @@ function buildChatDisplayList(messages, opts) {
|
|
|
5122
5933
|
} else if (grp.driver === "client") {
|
|
5123
5934
|
grp.finished = sawComplete || grp.status === "error" || grp.passCount >= MAX_INDEXING_RESUME_PASSES;
|
|
5124
5935
|
} else {
|
|
5125
|
-
grp.finished = !newestRunOfKey[order[oi]] || liveIndexChecked && !liveIndexKeys[grp.key];
|
|
5936
|
+
grp.finished = !newestRunOfKey[order[oi]] || !!doneKeys[grp.key] && !liveIndexKeys[grp.key] || liveIndexChecked && !liveIndexKeys[grp.key];
|
|
5126
5937
|
}
|
|
5127
5938
|
if (grp.status !== "done") {
|
|
5128
5939
|
grp.resolving = false;
|
|
5129
|
-
} else if (grp.mayHaveOlder && loadingOlderHistory && !liveIndexKeys[grp.key] && newestRunOfKey[order[oi]]) {
|
|
5940
|
+
} else if (grp.mayHaveOlder && loadingOlderHistory && !liveIndexKeys[grp.key] && !doneKeys[grp.key] && newestRunOfKey[order[oi]]) {
|
|
5130
5941
|
grp.resolving = true;
|
|
5131
5942
|
grp.resolvingReason = "history";
|
|
5132
5943
|
} else if (!grp.finished && grp.driver === "worker" && !liveIndexChecked && !liveIndexKeys[grp.key]) {
|
|
@@ -5136,18 +5947,130 @@ function buildChatDisplayList(messages, opts) {
|
|
|
5136
5947
|
grp.resolving = false;
|
|
5137
5948
|
}
|
|
5138
5949
|
}
|
|
5950
|
+
var stubList = [];
|
|
5951
|
+
var runStubs = opts && opts.runStubs;
|
|
5952
|
+
if (runStubs) {
|
|
5953
|
+
var coveredPaths = {};
|
|
5954
|
+
var coveredPathlessNames = {};
|
|
5955
|
+
for (var ci = 0; ci < order.length; ci++) {
|
|
5956
|
+
var cg = groups[order[ci]];
|
|
5957
|
+
if (cg.path) {
|
|
5958
|
+
coveredPaths[cg.path] = true;
|
|
5959
|
+
if (cg.key) coveredPaths[cg.key] = true;
|
|
5960
|
+
} else if (cg.name) coveredPathlessNames[cg.name] = true;
|
|
5961
|
+
else if (cg.key) coveredPaths[cg.key] = true;
|
|
5962
|
+
}
|
|
5963
|
+
var now = opts && typeof opts.now === "number" ? opts.now : Date.now();
|
|
5964
|
+
var stubClearedAt = opts && typeof opts.stubClearedAt === "number" && opts.stubClearedAt > 0 ? opts.stubClearedAt : 0;
|
|
5965
|
+
for (var sp in runStubs) {
|
|
5966
|
+
var rec = runStubs[sp];
|
|
5967
|
+
if (!sp || !rec || !rec.status || coveredPaths[sp]) continue;
|
|
5968
|
+
var fname = rec.filename || sp.split("/").pop() || sp;
|
|
5969
|
+
if (coveredPathlessNames[fname]) continue;
|
|
5970
|
+
if (stubPlatform && rec.platform && rec.platform !== stubPlatform) continue;
|
|
5971
|
+
var live = !!liveIndexKeys[sp] || !!liveIndexKeys[fname];
|
|
5972
|
+
var recWhen = typeof rec.finished === "number" ? rec.finished : typeof rec.started === "number" ? rec.started : void 0;
|
|
5973
|
+
if (stubClearedAt && !live && recWhen !== void 0 && recWhen <= stubClearedAt) continue;
|
|
5974
|
+
var st = "active";
|
|
5975
|
+
var fin = false;
|
|
5976
|
+
var res = false;
|
|
5977
|
+
var reason;
|
|
5978
|
+
if (!live) {
|
|
5979
|
+
if (rec.status === "done" || doneKeys[sp] || doneKeys[fname]) {
|
|
5980
|
+
st = "done";
|
|
5981
|
+
fin = true;
|
|
5982
|
+
} else if (rec.status === "error") {
|
|
5983
|
+
st = "error";
|
|
5984
|
+
fin = true;
|
|
5985
|
+
} else if (rec.status === "cancelled") {
|
|
5986
|
+
st = "cancelled";
|
|
5987
|
+
fin = true;
|
|
5988
|
+
} else if (liveIndexChecked) {
|
|
5989
|
+
st = "done";
|
|
5990
|
+
fin = true;
|
|
5991
|
+
} else if (typeof rec.started === "number" && now - rec.started > RUN_RECORD_WORKING_STALE_MS) {
|
|
5992
|
+
st = "error";
|
|
5993
|
+
fin = true;
|
|
5994
|
+
} else {
|
|
5995
|
+
res = true;
|
|
5996
|
+
reason = "status";
|
|
5997
|
+
}
|
|
5998
|
+
}
|
|
5999
|
+
var sg = {
|
|
6000
|
+
key: sp,
|
|
6001
|
+
// ONE identity for the run whether it renders from the record or
|
|
6002
|
+
// from its loaded passes: the views key the DOM off runKey, so a
|
|
6003
|
+
// 'stub:'-prefixed key meant every handoff was an unmount plus a
|
|
6004
|
+
// remount somewhere else. Named after the record's start, which
|
|
6005
|
+
// the real group below reuses when it has one.
|
|
6006
|
+
runKey: "run:" + sp + "#" + (typeof rec.started === "number" ? rec.started : "n"),
|
|
6007
|
+
name: fname,
|
|
6008
|
+
path: sp,
|
|
6009
|
+
mime: void 0,
|
|
6010
|
+
size: void 0,
|
|
6011
|
+
isReindex: false,
|
|
6012
|
+
members: [],
|
|
6013
|
+
passCount: 0,
|
|
6014
|
+
status: st,
|
|
6015
|
+
cancellableIds: [],
|
|
6016
|
+
cancelling: false,
|
|
6017
|
+
stopped: st === "cancelled",
|
|
6018
|
+
mayHaveOlder: hasMoreHistory,
|
|
6019
|
+
anchorIndex: -1,
|
|
6020
|
+
anchorId: "",
|
|
6021
|
+
visibleMembers: [],
|
|
6022
|
+
driver: !isPagedReadFile(fname, void 0) ? "single" : isImageVisionFile(fname, void 0) ? "worker" : windowedIndexing ? "worker" : "client",
|
|
6023
|
+
finished: fin,
|
|
6024
|
+
resolving: res,
|
|
6025
|
+
resolvingReason: reason,
|
|
6026
|
+
stub: true,
|
|
6027
|
+
stubError: rec.error || (st === "error" && !rec.error ? "Indexing did not finish." : void 0)
|
|
6028
|
+
};
|
|
6029
|
+
stubList.push({ started: typeof rec.started === "number" ? rec.started : Infinity, group: sg });
|
|
6030
|
+
}
|
|
6031
|
+
}
|
|
6032
|
+
var suppressAnchor = {};
|
|
6033
|
+
if (runStubs) {
|
|
6034
|
+
for (var ti2 = 0; ti2 < order.length; ti2++) {
|
|
6035
|
+
var tg = groups[order[ti2]];
|
|
6036
|
+
if (!newestRunOfKey[order[ti2]]) continue;
|
|
6037
|
+
var trec = tg.path && runStubs[tg.path] || runStubs[tg.key];
|
|
6038
|
+
if (!trec || typeof trec.started !== "number") continue;
|
|
6039
|
+
if (stubPlatform && trec.platform && trec.platform !== stubPlatform) continue;
|
|
6040
|
+
suppressAnchor[order[ti2]] = true;
|
|
6041
|
+
tg.runKey = "run:" + (tg.path || tg.key) + "#" + trec.started;
|
|
6042
|
+
stubList.push({ started: trec.started, group: tg });
|
|
6043
|
+
}
|
|
6044
|
+
}
|
|
6045
|
+
stubList.sort(function(a, b) {
|
|
6046
|
+
return a.started - b.started;
|
|
6047
|
+
});
|
|
5139
6048
|
var out = [];
|
|
6049
|
+
var si = 0;
|
|
5140
6050
|
for (var j = 0; j < list.length; j++) {
|
|
6051
|
+
var mts = list[j] && typeof list[j]._ts === "number" ? list[j]._ts : void 0;
|
|
6052
|
+
if (mts !== void 0) {
|
|
6053
|
+
while (si < stubList.length && stubList[si].started <= mts) {
|
|
6054
|
+
out.push({ kind: "indexing", group: stubList[si].group, index: -1 - si });
|
|
6055
|
+
si++;
|
|
6056
|
+
}
|
|
6057
|
+
}
|
|
5141
6058
|
var r = runOfIndex[j];
|
|
5142
6059
|
if (r === void 0) {
|
|
5143
6060
|
out.push({ kind: "message", msg: list[j], index: j });
|
|
5144
6061
|
continue;
|
|
5145
6062
|
}
|
|
5146
|
-
if (groups[r].anchorIndex === j
|
|
6063
|
+
if (groups[r].anchorIndex === j && !suppressAnchor[r]) {
|
|
6064
|
+
out.push({ kind: "indexing", group: groups[r], index: j });
|
|
6065
|
+
}
|
|
6066
|
+
}
|
|
6067
|
+
while (si < stubList.length) {
|
|
6068
|
+
out.push({ kind: "indexing", group: stubList[si].group, index: -1 - si });
|
|
6069
|
+
si++;
|
|
5147
6070
|
}
|
|
5148
6071
|
return out;
|
|
5149
6072
|
}
|
|
5150
6073
|
|
|
5151
|
-
export { BG_INDEXING_QUEUE_SUFFIX, BOM, BOM_EXTS, CLAUDE_INPUT_CAP_RATIO, CLAUDE_PER_REQUEST_INPUT_CAP, CONTEXT_WINDOW_BY_MODEL, CONTEXT_WINDOW_DEFAULT, ChatSession, DEFAULT_CLAUDE_MODEL, DEFAULT_OPENAI_MODEL, EMPTY_INDEXING_REPLY, EXPIRED_ATTACHMENT_URL_HOST, EXPIRED_ATTACHMENT_URL_ORIGIN, EXPIRED_LINK_REFRESH_EXPIRES_SECONDS, EXT_CONTENT_TYPES, HISTORY_BUDGET_RATIO, HISTORY_FILL_SLACK_PX, HISTORY_TOKEN_BUDGET, HTML_EXTS, HTML_HEAD_WINDOW, IMAGE_PREVIEWS_PER_MESSAGE, INDEXING_COMPLETE_MARKER, INLINE_LINK_GLYPH, INLINE_LINK_UNAVAILABLE_GLYPH, INLINE_LINK_UNAVAILABLE_SUFFIX, LINK_LABEL_MAX_DISPLAY_CHARS, LINK_REFRESH_WINDOW_MS, MAX_CONCURRENT_BG_POLLS, MAX_HISTORY_FILL_PAGES, MAX_HISTORY_MESSAGES, MAX_PARSED_CONTENT_CHARS, MCP_NAME, MIN_INPUT_TOKEN_BUDGET, OUTPUT_TOKEN_RESERVE, POLL_INTERVAL, PREVIEWABLE_IMAGE_CONTENT_TYPES, PREVIEW_BROWSER_CACHE_SECONDS, RENDER_FROM_TOKEN, RTF_EXTS, TOOL_AND_RESPONSE_BUFFER, XML_EXTS, applyEncodingDeclaration, bgIndexingQueueName, buildAiAgentValue, buildBoundedChatMessages, buildChatDisplayList, buildChatSystemPrompt, buildDisplayExpiredAttachmentHref, buildIndexingContinueMessage, buildIndexingRenderContinueTemplate, buildIndexingRenderMessage, buildIndexingSystemPrompt, buildIndexingUserMessage, buildIndexingWindowMessage, callClaudeWithMcp, callClaudeWithPublicMcp, callOpenAIWithPublicMcp, chatEngineConfig, classifyInlineLink, clearAttachmentParsers, clearImagePreviewCache, composeUserMessage, configureChatEngine, contentTypeForExt, createHistoryFiller, createInlineLinkRegex, encodePathSegments, encodingClassForExt, ensureHtmlCharset, ensureXmlEncoding, escapeInlineHtml, escapeRtfNonAscii, estimateMessageTokens, estimateTextTokens, extOf, extractClaudeText, extractLastUserTextFromRequest, extractOpenAIText, extractRemotePathFromAttachmentHref, fillHistoryViewport, filterListByClearHorizon, findAttachmentParser, formatChatTimestamp, getAttachmentParsers, getChatHistory, getContextWindow, getErrorMessage, getExpiredAttachmentVisiblePath, getProjectContextWindow, getVisionProfile, groupAttachmentFailures, hasBom, hydrateImagePreviews, isAuthExpiredError, isBgIndexingQueue, isErrorResponseBody, isHttpUrlLike, isIndexingRequestText, isLinkUnavailable, isNonRetryableRequestError, isOfficeFile, isPreviewableImagePath, isServerExtractable, isServiceDbAttachmentHref, linkUnavailableKeyForHref, linkUnavailableKeyForPath, listClaudeModels, listOpenAIModels, looksLikeRtf, makeExtractPlaceholder, mapHistoryListToMessages, markImagePreviewStale, needsBomForExt, normalizeAttachmentPathCandidate, normalizeExt, normalizeTextContent, normalizeTrailingInlineToken, notifyAgentSaveAttachment, parseAiAgentValue, parseAttachmentContent, parseIndexingLabel, parseIndexingRequestText, peekImagePreviewUrl, prepareDownloadText, previewImageContentType, previewableExtOf, readExpiredAttachmentHref, registerAttachmentParser, registerModelContextWindows, renderInlineLinkHtml, repairUrlEntities, repairUrlWhitespace, resolveImagePreviewUrl, safeDecodeURIComponent, sanitizeAttachmentLinksForHistory, setProjectContextWindow, stripFileBlocksFromHistory, transformContentWithImages, transformContentWithOpenAIImages, truncateLabelForDisplay, wallClockNow };
|
|
6074
|
+
export { BG_INDEXING_QUEUE_SUFFIX, BOM, BOM_EXTS, CLAUDE_INPUT_CAP_RATIO, CLAUDE_PER_REQUEST_INPUT_CAP, CONTEXT_WINDOW_BY_MODEL, CONTEXT_WINDOW_DEFAULT, ChatSession, DEFAULT_CLAUDE_MODEL, DEFAULT_CONTEXT_WINDOW, DEFAULT_OPENAI_MODEL, EMPTY_INDEXING_REPLY, EXPIRED_ATTACHMENT_URL_HOST, EXPIRED_ATTACHMENT_URL_ORIGIN, EXPIRED_LINK_REFRESH_EXPIRES_SECONDS, EXT_CONTENT_TYPES, HISTORY_BUDGET_RATIO, HISTORY_FILL_SLACK_PX, HISTORY_TOKEN_BUDGET, HTML_EXTS, HTML_HEAD_WINDOW, IMAGE_PREVIEWS_PER_MESSAGE, INDEXING_COMPLETE_MARKER, INLINE_LINK_GLYPH, INLINE_LINK_UNAVAILABLE_GLYPH, INLINE_LINK_UNAVAILABLE_SUFFIX, INPUT_CAP_RATIO, LINK_LABEL_MAX_DISPLAY_CHARS, LINK_REFRESH_WINDOW_MS, MAX_CONCURRENT_BG_POLLS, MAX_HISTORY_FILL_PAGES, MAX_HISTORY_MESSAGES, MAX_OUTPUT_BY_MODEL, MAX_OUTPUT_TOKENS, MAX_PARSED_CONTENT_CHARS, MCP_NAME, MIN_INPUT_TOKEN_BUDGET, MIN_PER_REQUEST_INPUT_CAP, OUTPUT_TOKEN_RESERVE, POLL_INTERVAL, PREVIEWABLE_IMAGE_CONTENT_TYPES, PREVIEW_BROWSER_CACHE_SECONDS, RENDER_FROM_TOKEN, RTF_EXTS, RUN_RECORD_WORKING_STALE_MS, TOOL_AND_RESPONSE_BUFFER, XML_EXTS, __resetSplitHistoryState, applyEncodingDeclaration, bgIndexingQueueName, buildAiAgentValue, buildBoundedChatMessages, buildChatDisplayList, buildChatSystemPrompt, buildDisplayExpiredAttachmentHref, buildHistoryItemFullId, buildIndexingContinueMessage, buildIndexingRenderContinueTemplate, buildIndexingRenderMessage, buildIndexingSystemPrompt, buildIndexingUserMessage, buildIndexingWindowMessage, callClaudeWithMcp, callClaudeWithPublicMcp, callOpenAIWithPublicMcp, chatEngineConfig, classifyInlineLink, clearAttachmentParsers, clearImagePreviewCache, composeUserMessage, configureChatEngine, contentTypeForExt, createHistoryFiller, createInlineLinkRegex, encodePathSegments, encodingClassForExt, ensureHtmlCharset, ensureXmlEncoding, escapeInlineHtml, escapeRtfNonAscii, estimateMessageTokens, estimateTextTokens, extOf, extractClaudeText, extractLastUserTextFromRequest, extractOpenAIText, extractRemotePathFromAttachmentHref, fetchLiveIndexingKeys, fillHistoryViewport, filterListByClearHorizon, findAttachmentParser, formatChatTimestamp, getAttachmentParsers, getChatHistory, getContextWindow, getErrorMessage, getExpiredAttachmentVisiblePath, getInputTokenBudget, getMaxOutputTokens, getModelContextWindow, getProjectContextWindow, getSplitChatHistory, getVisionProfile, groupAttachmentFailures, hasBom, hydrateImagePreviews, indexDoneUniqueId, isAuthExpiredError, isBgIndexingQueue, isErrorResponseBody, isHttpUrlLike, isIndexingRequestText, isLinkUnavailable, isNonRetryableRequestError, isOfficeFile, isPreviewableImagePath, isProviderApiKeyError, isServerExtractable, isServiceDbAttachmentHref, linkUnavailableKeyForHref, linkUnavailableKeyForPath, listClaudeModels, listOpenAIModels, looksLikeRtf, makeExtractPlaceholder, mapHistoryListToMessages, markImagePreviewStale, needsBomForExt, normalizeAttachmentPathCandidate, normalizeExt, normalizeTextContent, normalizeTrailingInlineToken, notifyAgentSaveAttachment, parseAiAgentValue, parseAttachmentContent, parseIndexingLabel, parseIndexingRequestText, peekImagePreviewUrl, prepareDownloadText, previewImageContentType, previewableExtOf, readExpiredAttachmentHref, registerAttachmentParser, registerModelContextWindows, renderInlineLinkHtml, repairUrlEntities, repairUrlWhitespace, resolveImagePreviewUrl, runIndexUniqueId, safeDecodeURIComponent, sanitizeAttachmentLinksForHistory, setProjectContextWindow, stripFileBlocksFromHistory, transformContentWithImages, transformContentWithOpenAIImages, truncateLabelForDisplay, upsertIndexRunRecordSafe, wallClockNow };
|
|
5152
6075
|
//# sourceMappingURL=engine.mjs.map
|
|
5153
6076
|
//# sourceMappingURL=engine.mjs.map
|