forge-openclaw-plugin 0.2.114 → 0.2.116

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,7 @@ import { closeSync, existsSync, mkdirSync, openSync } from "node:fs";
3
3
  import net from "node:net";
4
4
  import { homedir } from "node:os";
5
5
  import path from "node:path";
6
- import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
6
+ import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
7
7
  import { fileURLToPath } from "node:url";
8
8
  const LOCAL_HOSTNAMES = new Set(["127.0.0.1", "localhost", "::1"]);
9
9
  const STARTUP_TIMEOUT_MS = 15_000;
@@ -37,6 +37,12 @@ function getRuntimeStatePath(config) {
37
37
  const origin = new URL(config.origin).hostname.toLowerCase().replace(/[^a-z0-9._-]+/g, "-");
38
38
  return path.join(homedir(), ".openclaw", "run", FORGE_PLUGIN_ID, `${origin}-${config.port}.json`);
39
39
  }
40
+ function getRuntimeStateDir() {
41
+ return path.join(homedir(), ".openclaw", "run", FORGE_PLUGIN_ID);
42
+ }
43
+ function getRuntimeStateOrigin(config) {
44
+ return new URL(config.origin).hostname.toLowerCase().replace(/[^a-z0-9._-]+/g, "-");
45
+ }
40
46
  function getPreferredPortStatePath(origin) {
41
47
  const hostname = new URL(origin).hostname.toLowerCase().replace(/[^a-z0-9._-]+/g, "-");
42
48
  return path.join(homedir(), ".openclaw", "run", FORGE_PLUGIN_ID, `${hostname}-preferred-port.json`);
@@ -118,6 +124,10 @@ async function writeRuntimeState(config, pid) {
118
124
  async function clearRuntimeState(config) {
119
125
  await rm(getRuntimeStatePath(config), { force: true });
120
126
  }
127
+ async function clearRuntimeStateForState(state) {
128
+ const origin = new URL(state.origin).hostname.toLowerCase().replace(/[^a-z0-9._-]+/g, "-");
129
+ await rm(path.join(getRuntimeStateDir(), `${origin}-${state.port}.json`), { force: true });
130
+ }
121
131
  async function readRuntimeState(config) {
122
132
  try {
123
133
  const payload = await readFile(getRuntimeStatePath(config), "utf8");
@@ -138,6 +148,52 @@ async function readRuntimeState(config) {
138
148
  return null;
139
149
  }
140
150
  }
151
+ async function readRuntimeStateFile(filePath) {
152
+ try {
153
+ const payload = await readFile(filePath, "utf8");
154
+ const parsed = JSON.parse(payload);
155
+ if (typeof parsed.pid !== "number" ||
156
+ !Number.isFinite(parsed.pid) ||
157
+ typeof parsed.origin !== "string" ||
158
+ typeof parsed.port !== "number" ||
159
+ !Number.isFinite(parsed.port)) {
160
+ return null;
161
+ }
162
+ return {
163
+ pid: Math.trunc(parsed.pid),
164
+ origin: parsed.origin,
165
+ port: Math.trunc(parsed.port),
166
+ baseUrl: typeof parsed.baseUrl === "string" ? parsed.baseUrl : buildForgeBaseUrl(parsed.origin, parsed.port),
167
+ startedAt: typeof parsed.startedAt === "string" ? parsed.startedAt : new Date(0).toISOString(),
168
+ logPath: typeof parsed.logPath === "string" ? parsed.logPath : null
169
+ };
170
+ }
171
+ catch {
172
+ return null;
173
+ }
174
+ }
175
+ async function readRuntimeStatesForOrigin(config) {
176
+ const stateDir = getRuntimeStateDir();
177
+ const origin = getRuntimeStateOrigin(config);
178
+ let entries;
179
+ try {
180
+ entries = await readdir(stateDir);
181
+ }
182
+ catch {
183
+ return [];
184
+ }
185
+ const states = [];
186
+ for (const entry of entries) {
187
+ if (!entry.startsWith(`${origin}-`) || !entry.endsWith(".json") || entry.endsWith("-preferred-port.json")) {
188
+ continue;
189
+ }
190
+ const state = await readRuntimeStateFile(path.join(stateDir, entry));
191
+ if (state) {
192
+ states.push(state);
193
+ }
194
+ }
195
+ return states;
196
+ }
141
197
  function processExists(pid) {
142
198
  try {
143
199
  process.kill(pid, 0);
@@ -147,6 +203,34 @@ function processExists(pid) {
147
203
  return !(error instanceof Error) || !("code" in error) || error.code !== "ESRCH";
148
204
  }
149
205
  }
206
+ async function cleanupSupersededManagedRuntimes(config, expectedDataRoot) {
207
+ const states = await readRuntimeStatesForOrigin(config);
208
+ for (const state of states) {
209
+ if (state.port === config.port) {
210
+ continue;
211
+ }
212
+ if (!processExists(state.pid)) {
213
+ await clearRuntimeStateForState(state);
214
+ continue;
215
+ }
216
+ const alternateConfig = {
217
+ ...config,
218
+ port: state.port,
219
+ baseUrl: buildForgeBaseUrl(state.origin, state.port),
220
+ webAppUrl: buildForgeWebAppUrl(state.origin, state.port)
221
+ };
222
+ const alternateProbe = await probeForgeRuntime(alternateConfig, HEALTHCHECK_TIMEOUT_MS);
223
+ if (!alternateProbe.healthy || !isExpectedDataRoot(expectedDataRoot, alternateProbe.storageRoot)) {
224
+ continue;
225
+ }
226
+ process.kill(state.pid, "SIGTERM");
227
+ if (!(await waitForProcessExit(state.pid, 5_000))) {
228
+ process.kill(state.pid, "SIGKILL");
229
+ await waitForProcessExit(state.pid, 2_000);
230
+ }
231
+ await clearRuntimeStateForState(state);
232
+ }
233
+ }
150
234
  async function waitForProcessExit(pid, timeoutMs) {
151
235
  const deadline = Date.now() + timeoutMs;
152
236
  while (Date.now() < deadline) {
@@ -490,6 +574,7 @@ export async function ensureForgeRuntimeReady(config) {
490
574
  const expectedDataRoot = getExpectedDataRoot(config);
491
575
  const initialProbe = await probeForgeRuntime(config, HEALTHCHECK_TIMEOUT_MS);
492
576
  if (initialProbe.healthy && isExpectedDataRoot(expectedDataRoot, initialProbe.storageRoot)) {
577
+ await cleanupSupersededManagedRuntimes(config, expectedDataRoot);
493
578
  const existingState = await readRuntimeState(config);
494
579
  if (!existingState) {
495
580
  await adoptManagedRuntimeState(config, initialProbe);
@@ -0,0 +1,12 @@
1
+ UPDATE movement_stays
2
+ SET published_note_id = NULL
3
+ WHERE published_note_id IS NOT NULL;
4
+
5
+ UPDATE movement_trips
6
+ SET published_note_id = NULL
7
+ WHERE published_note_id IS NOT NULL;
8
+
9
+ DELETE FROM notes
10
+ WHERE source = 'system'
11
+ AND kind = 'evidence'
12
+ AND json_extract(frontmatter_json, '$.movement.kind') IN ('stay', 'trip');
@@ -9179,7 +9179,8 @@ export async function buildServer(options = {}) {
9179
9179
  pages: listWikiPages({
9180
9180
  spaceId: query.spaceId,
9181
9181
  kind: query.kind,
9182
- limit: query.limit ? Number(query.limit) : undefined
9182
+ limit: query.limit ? Number(query.limit) : undefined,
9183
+ includeHidden: query.includeHidden === "true"
9183
9184
  })
9184
9185
  };
9185
9186
  });
@@ -3,7 +3,7 @@ import os from "node:os";
3
3
  import { promisify } from "node:util";
4
4
  import bonjourService from "bonjour-service";
5
5
  import { logForgeDebug } from "./debug.js";
6
- import { companionIrohApiBaseUrlFromNodeId, companionIrohUiBaseUrlFromNodeId, getCompanionIrohStatus } from "./services/companion-iroh.js";
6
+ import { getCompanionIrohStatus } from "./services/companion-iroh.js";
7
7
  const execFileAsync = promisify(execFile);
8
8
  const BonjourConstructor = bonjourService.Bonjour ??
9
9
  bonjourService.default ??
@@ -37,12 +37,8 @@ export async function startForgeDiscoveryAdvertiser(options) {
37
37
  tsApiBaseUrl: tailscaleTargets.apiBaseUrl ?? "",
38
38
  tsUiBaseUrl: tailscaleTargets.uiBaseUrl ?? "",
39
39
  tsDnsName: tailscaleTargets.dnsName ?? "",
40
- irohApiBaseUrl: irohNodeId
41
- ? companionIrohApiBaseUrlFromNodeId(irohNodeId)
42
- : "",
43
- irohUiBaseUrl: irohNodeId
44
- ? companionIrohUiBaseUrlFromNodeId(irohNodeId)
45
- : "",
40
+ irohApiBaseUrl: irohNodeId ? tailscaleTargets.apiBaseUrl ?? "" : "",
41
+ irohUiBaseUrl: irohNodeId ? tailscaleTargets.uiBaseUrl ?? "" : "",
46
42
  irohProvider: irohNodeId ? "forge-companion-iroh" : "",
47
43
  irohNodeId: irohNodeId ?? "",
48
44
  irohRelay: irohTransport.pairPayload?.relay ?? "",
@@ -20,8 +20,11 @@ const MODEL_CONTEXT_WINDOWS = {
20
20
  };
21
21
  const DEFAULT_CONTEXT_WINDOW = 400_000;
22
22
  const RESERVED_RESPONSE_TOKENS = 140_000;
23
+ const CODEX_WIKI_COMPILE_CONTEXT_WINDOW = 120_000;
24
+ const CODEX_WIKI_COMPILE_RESERVED_RESPONSE_TOKENS = 60_000;
23
25
  const APPROX_CHARS_PER_TOKEN = 4;
24
26
  const REQUEST_TIMEOUT_MS = 90_000;
27
+ const CODEX_FOREGROUND_COMPILE_TIMEOUT_MS = 10 * 60_000;
25
28
  const BACKGROUND_POLL_INTERVAL_MS = 2_000;
26
29
  const DEFAULT_CODEX_BASE_URL = "https://chatgpt.com/backend-api";
27
30
  const CODEX_JWT_CLAIM_PATH = "https://api.openai.com/auth";
@@ -164,6 +167,106 @@ function parseOutputText(payload) {
164
167
  }
165
168
  return null;
166
169
  }
170
+ function buildOutputTextPayload(text) {
171
+ return {
172
+ status: "completed",
173
+ output: [
174
+ {
175
+ content: [{ type: "output_text", text }]
176
+ }
177
+ ]
178
+ };
179
+ }
180
+ function parseCodexEventStreamPayload(streamText) {
181
+ const chunks = [];
182
+ let latestResponse = null;
183
+ let failedError = null;
184
+ let dataLines = [];
185
+ const flushEvent = () => {
186
+ if (dataLines.length === 0) {
187
+ return;
188
+ }
189
+ const raw = dataLines.join("\n");
190
+ dataLines = [];
191
+ if (raw.trim() === "[DONE]") {
192
+ return;
193
+ }
194
+ let payload;
195
+ try {
196
+ payload = JSON.parse(raw);
197
+ }
198
+ catch {
199
+ return;
200
+ }
201
+ const payloadType = typeof payload.type === "string" ? payload.type : null;
202
+ if (payloadType === "response.output_text.delta") {
203
+ if (typeof payload.delta === "string") {
204
+ chunks.push(payload.delta);
205
+ }
206
+ return;
207
+ }
208
+ if (payloadType === "response.output_text.done") {
209
+ if (typeof payload.text === "string" && chunks.length === 0) {
210
+ chunks.push(payload.text);
211
+ }
212
+ return;
213
+ }
214
+ if (payloadType === "response.completed" ||
215
+ payloadType === "response.failed") {
216
+ const response = payload.response;
217
+ if (response && typeof response === "object") {
218
+ latestResponse = response;
219
+ const text = parseOutputText(latestResponse);
220
+ if (text && chunks.length === 0) {
221
+ chunks.push(text);
222
+ }
223
+ }
224
+ if (payloadType === "response.failed") {
225
+ failedError =
226
+ payload.error ??
227
+ latestResponse?.error ??
228
+ "Codex response failed.";
229
+ }
230
+ }
231
+ };
232
+ for (const line of streamText.split(/\r?\n/)) {
233
+ if (line === "") {
234
+ flushEvent();
235
+ continue;
236
+ }
237
+ if (line.startsWith(":")) {
238
+ continue;
239
+ }
240
+ if (line.startsWith("data:")) {
241
+ dataLines.push(line.slice("data:".length).trimStart());
242
+ }
243
+ }
244
+ flushEvent();
245
+ if (failedError) {
246
+ throw new Error(`Codex response failed: ${JSON.stringify(failedError)}`);
247
+ }
248
+ const text = chunks.join("");
249
+ if (latestResponse) {
250
+ if (!parseOutputText(latestResponse) && text) {
251
+ return buildOutputTextPayload(text);
252
+ }
253
+ return latestResponse;
254
+ }
255
+ if (text) {
256
+ return buildOutputTextPayload(text);
257
+ }
258
+ try {
259
+ return JSON.parse(streamText);
260
+ }
261
+ catch {
262
+ return buildOutputTextPayload("");
263
+ }
264
+ }
265
+ async function readProviderPayload(response, profile) {
266
+ return isCodexProfile(profile)
267
+ ? parseCodexEventStreamPayload(await response.text())
268
+ : readJsonPayload(response);
269
+ }
167
270
  function readReasoningEffort(profile) {
168
271
  return typeof profile.metadata.reasoningEffort === "string"
169
272
  ? profile.metadata.reasoningEffort
@@ -233,6 +336,7 @@ function buildRequestHeaders(profile, apiKey, options = {}) {
233
336
  headers["OpenAI-Beta"] = "responses=experimental";
234
337
  headers.originator = "pi";
235
338
  headers["chatgpt-account-id"] = extractCodexAccountId(apiKey);
339
+ headers.accept = "text/event-stream";
236
340
  return headers;
237
341
  }
238
342
  function buildReasoningConfiguration(profile) {
@@ -250,12 +354,21 @@ function buildTextConfiguration(options) {
250
354
  }
251
355
  return Object.keys(text).length > 0 ? text : undefined;
252
356
  }
357
+ function buildInstructionsPayload(profile, instructions) {
358
+ return isCodexProfile(profile) ? { instructions } : {};
359
+ }
253
360
  function estimateTokens(text) {
254
361
  return Math.ceil(text.length / APPROX_CHARS_PER_TOKEN);
255
362
  }
256
363
  function computeSourceExcerpt(profile, sourceText) {
257
- const contextWindow = MODEL_CONTEXT_WINDOWS[profile.model] ?? DEFAULT_CONTEXT_WINDOW;
258
- const inputBudget = Math.max(16_000, contextWindow - RESERVED_RESPONSE_TOKENS);
364
+ const configuredContextWindow = MODEL_CONTEXT_WINDOWS[profile.model] ?? DEFAULT_CONTEXT_WINDOW;
365
+ const contextWindow = isCodexProfile(profile)
366
+ ? Math.min(configuredContextWindow, CODEX_WIKI_COMPILE_CONTEXT_WINDOW)
367
+ : configuredContextWindow;
368
+ const reservedResponseTokens = isCodexProfile(profile)
369
+ ? CODEX_WIKI_COMPILE_RESERVED_RESPONSE_TOKENS
370
+ : RESERVED_RESPONSE_TOKENS;
371
+ const inputBudget = Math.max(16_000, contextWindow - reservedResponseTokens);
259
372
  const estimatedTokens = estimateTokens(sourceText);
260
373
  if (estimatedTokens <= inputBudget) {
261
374
  return {
@@ -359,7 +472,11 @@ export class OpenAiResponsesProvider {
359
472
  }),
360
473
  body: JSON.stringify({
361
474
  model: profile.model,
362
- input: "Reply with the single word ok.",
475
+ ...buildInstructionsPayload(profile, "Reply with the single word ok."),
476
+ input: isCodexProfile(profile)
477
+ ? "Connection test."
478
+ : "Reply with the single word ok.",
479
+ ...(isCodexProfile(profile) ? { stream: true, store: false } : {}),
363
480
  max_output_tokens: 24,
364
481
  reasoning: buildReasoningConfiguration(profile),
365
482
  text: buildTextConfiguration({ profile })
@@ -404,7 +521,7 @@ export class OpenAiResponsesProvider {
404
521
  });
405
522
  throw new Error(`OpenAI connection test failed (${response.status})${message ? `: ${message}` : ""}`);
406
523
  }
407
- const payload = await readJsonPayload(response);
524
+ const payload = await readProviderPayload(response, profile);
408
525
  emitDiagnostic(logger, {
409
526
  level: "info",
410
527
  message: "OpenAI connection test completed.",
@@ -440,20 +557,31 @@ export class OpenAiResponsesProvider {
440
557
  }),
441
558
  body: JSON.stringify({
442
559
  model: profile.model,
443
- input: [
444
- ...(systemPrompt?.trim()
445
- ? [
446
- {
447
- role: "system",
448
- content: [{ type: "input_text", text: systemPrompt.trim() }]
449
- }
450
- ]
451
- : []),
452
- {
453
- role: "user",
454
- content: [{ type: "input_text", text: prompt }]
455
- }
456
- ],
560
+ ...buildInstructionsPayload(profile, systemPrompt?.trim() || "Follow the user's request."),
561
+ input: isCodexProfile(profile)
562
+ ? [
563
+ {
564
+ role: "user",
565
+ content: [{ type: "input_text", text: prompt }]
566
+ }
567
+ ]
568
+ : [
569
+ ...(systemPrompt?.trim()
570
+ ? [
571
+ {
572
+ role: "system",
573
+ content: [
574
+ { type: "input_text", text: systemPrompt.trim() }
575
+ ]
576
+ }
577
+ ]
578
+ : []),
579
+ {
580
+ role: "user",
581
+ content: [{ type: "input_text", text: prompt }]
582
+ }
583
+ ],
584
+ ...(isCodexProfile(profile) ? { stream: true, store: false } : {}),
457
585
  reasoning: buildReasoningConfiguration(profile),
458
586
  text: buildTextConfiguration({ profile }),
459
587
  max_output_tokens: 1200
@@ -463,7 +591,7 @@ export class OpenAiResponsesProvider {
463
591
  const message = await response.text();
464
592
  throw new Error(`OpenAI text prompt failed (${response.status})${message ? `: ${message}` : ""}`);
465
593
  }
466
- const payload = await readJsonPayload(response);
594
+ const payload = await readProviderPayload(response, profile);
467
595
  return {
468
596
  outputText: parseOutputText(payload)?.trim() || ""
469
597
  };
@@ -477,16 +605,16 @@ export class OpenAiResponsesProvider {
477
605
  "Goal:",
478
606
  "- Produce one main overview page in markdown.",
479
607
  "- Split durable subtopics into articleCandidates with full draft markdown.",
480
- "- Propose Forge entities only when the source clearly supports a durable record.",
608
+ "- Propose structured operational entities only when the source clearly supports a durable record.",
481
609
  "- Suggest page updates only when the source clearly belongs in an existing page instead of a new page.",
482
610
  "What Forge expects:",
483
611
  "- The main markdown should be a readable overview or anchor page, not a raw source dump.",
484
612
  "- articleCandidates should contain real draft wiki pages with their own markdown, not just titles.",
485
613
  "- entityProposals must use one of these entityType values only: goal, project, task, habit, strategy, psyche_value, note.",
486
614
  "- Use suggestedFields only for fields that truly fit the entity type. Use null for unknown scalar fields and [] for unknown list fields.",
487
- "- Keep entity proposals conservative. Prefer wiki pages when the source is informative but not actionable enough for a Forge entity.",
615
+ "- Keep structured operational entity proposals conservative. Prefer wiki page entities when the source is informative but not actionable enough for goals, projects, tasks, habits, strategies, values, or evidence notes.",
488
616
  "Forge ontology:",
489
- "- Forge has two durable knowledge surfaces: wiki pages and structured entities.",
617
+ "- Forge has two durable entity families: wiki page entities and structured operational entities.",
490
618
  "- Use wiki pages for rich context, summaries, explanations, relationships, timelines, source synthesis, and themes that are broader than one action item.",
491
619
  "- Use entities for operational objects that Forge can track directly: goals, projects, tasks, habits, strategies, psyche values, and durable notes.",
492
620
  "- When the same topic needs both explanation and operations, create both: a wiki page for context and an entity proposal for the operational record.",
@@ -498,11 +626,25 @@ export class OpenAiResponsesProvider {
498
626
  "- For chats and transcripts, extract the durable parts: people, relationships, ongoing projects, commitments, habits, values, decisions, questions, sources, and evidence.",
499
627
  "- Merge repetitive back-and-forth into concise summaries.",
500
628
  "- Use short quotes only when the exact phrase matters.",
629
+ "Sensitive information rules:",
630
+ "- Never store or reproduce secrets, passwords, passphrases, recovery codes, API keys, access tokens, refresh tokens, session cookies, private keys, seed phrases, one-time codes, full payment card numbers, or equivalent credentials.",
631
+ "- If the source contains a secret or credential, replace the value with [REDACTED SECRET] and keep only the minimum non-secret context needed to explain why it appeared.",
632
+ "- Do not create wiki pages, notes, entity proposals, aliases, tags, titles, or page update suggestions that preserve secret values.",
501
633
  "How to split pages:",
502
634
  "- Keep markdown as the overview page for this source.",
503
635
  "- If one topic deserves its own page, put it in articleCandidates with title, slug, summary, rationale, markdown, tags, aliases, and parentSlug.",
504
636
  "- Use parentSlug when the draft page clearly belongs under an existing Forge wiki branch such as people, projects, concepts, sources, or chronicle.",
637
+ "- For chats and message logs, create articleCandidates for durable people, family members, partners, collaborators, sources, places, projects, important dated events, recurring themes, and concepts when the source contains enough reusable context to make a useful page.",
638
+ "- Do not leave durable people only as [[links]] when the source gives relationship, role, timeline, or follow-up context; create a lightweight person page candidate instead.",
639
+ "- Do not leave important or specific events only as bullets in the overview when they have their own timeline, decision, consequence, open loop, or later follow-up value; create a chronicle page candidate instead.",
640
+ "- Do not leave recurring concepts only as inline observations when the source gives definitions, examples, tradeoffs, or repeated behavior patterns; create a concept page candidate instead.",
505
641
  "- Do not create articleCandidates for every minor mention; only create pages that would be useful to reopen later.",
642
+ "Coverage audit before output:",
643
+ "- Before final JSON, scan the source for page-worthy entities in these categories: people, family/partners, organizations, places, sources/files/links, dated events, decisions, recurring concepts, health/care episodes, projects/plans, and relationship contexts.",
644
+ "- For each category with enough durable context, either create an articleCandidate or make the reason for not splitting clear inside the overview page.",
645
+ "- For a close relationship chat, it is normal to create several page candidates: the source/export page, the main person page, close family/partner pages when named with context, and major event pages such as meetups, care crises, travel episodes, or important decisions.",
646
+ "- Prefer fewer high-quality pages over many stubs, but do not collapse distinct durable people or major events into one page merely because they came from the same file.",
647
+ "- If the source has a named spouse/partner, family member, collaborator, or recurring person with role/context/follow-up value, create a person page candidate even if the page is short.",
506
648
  "Useful high-level wiki themes:",
507
649
  "- people: people, collaborators, family, teams, roles, relationship context.",
508
650
  "- projects: bounded initiatives, active efforts, plans, milestones, workstreams.",
@@ -528,7 +670,7 @@ export class OpenAiResponsesProvider {
528
670
  "Entity proposal rules:",
529
671
  "- Never invent IDs.",
530
672
  "- Do not propose an entity just because it is mentioned once without durable importance.",
531
- "- People, concepts, sources, places, and broad life areas are usually wiki pages, not Forge entities.",
673
+ "- People, concepts, sources, places, and broad life areas are usually wiki page entities, not structured operational entity proposals.",
532
674
  "- Goals should be outcomes, not chores.",
533
675
  "- Projects should group multiple steps or phases, not a single errand.",
534
676
  "- Tasks should be concrete and actionable enough to do soon.",
@@ -617,6 +759,8 @@ export class OpenAiResponsesProvider {
617
759
  role: "user",
618
760
  content: userContent
619
761
  });
762
+ const requestInputs = isCodexProfile(profile) ? inputs.slice(1) : inputs;
763
+ const useStoredBackgroundResponse = !isCodexProfile(profile);
620
764
  let payload;
621
765
  let responseId = resumeResponseId?.trim() || null;
622
766
  if (responseId) {
@@ -667,11 +811,17 @@ export class OpenAiResponsesProvider {
667
811
  }),
668
812
  body: JSON.stringify({
669
813
  model: profile.model,
670
- input: inputs,
671
- store: true,
672
- background: true,
673
- prompt_cache_retention: profile.model === "gpt-5.4" ? "24h" : "in_memory",
674
- prompt_cache_key: `forge-wiki-ingest:${profile.model}:${input.parseStrategy}:${input.mimeType}`,
814
+ ...buildInstructionsPayload(profile, prompt),
815
+ input: requestInputs,
816
+ store: useStoredBackgroundResponse,
817
+ ...(isCodexProfile(profile) ? { stream: true } : {}),
818
+ ...(useStoredBackgroundResponse
819
+ ? {
820
+ background: true,
821
+ prompt_cache_retention: profile.model === "gpt-5.4" ? "24h" : "in_memory",
822
+ prompt_cache_key: `forge-wiki-ingest:${profile.model}:${input.parseStrategy}:${input.mimeType}`
823
+ }
824
+ : {}),
675
825
  reasoning: buildReasoningConfiguration(profile),
676
826
  text: buildTextConfiguration({
677
827
  profile,
@@ -683,7 +833,9 @@ export class OpenAiResponsesProvider {
683
833
  }
684
834
  })
685
835
  }),
686
- signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
836
+ signal: AbortSignal.timeout(isCodexProfile(profile)
837
+ ? CODEX_FOREGROUND_COMPILE_TIMEOUT_MS
838
+ : REQUEST_TIMEOUT_MS)
687
839
  });
688
840
  }
689
841
  catch (error) {
@@ -733,17 +885,21 @@ export class OpenAiResponsesProvider {
733
885
  });
734
886
  throw new Error(`LLM compilation failed: ${createResponse.status}${message ? `: ${message}` : ""}`);
735
887
  }
736
- payload = await readJsonPayload(createResponse);
888
+ payload = await readProviderPayload(createResponse, profile);
737
889
  responseId = readResponseId(payload);
738
- if (!responseId) {
890
+ if (useStoredBackgroundResponse && !responseId) {
739
891
  throw new Error("OpenAI background response did not include an id for polling.");
740
892
  }
741
893
  emitDiagnostic(logger, {
742
894
  level: "info",
743
- message: "OpenAI accepted the wiki compilation job for background processing.",
895
+ message: useStoredBackgroundResponse
896
+ ? "OpenAI accepted the wiki compilation job for background processing."
897
+ : "OpenAI Codex returned a foreground wiki compilation response.",
744
898
  details: {
745
899
  scope: "wiki_llm",
746
- eventKey: "llm_compile_background_started",
900
+ eventKey: useStoredBackgroundResponse
901
+ ? "llm_compile_background_started"
902
+ : "llm_compile_foreground_completed",
747
903
  provider: profile.provider,
748
904
  baseUrl: profile.baseUrl,
749
905
  model: profile.model,
@@ -755,7 +911,8 @@ export class OpenAiResponsesProvider {
755
911
  }
756
912
  let pollCount = 0;
757
913
  let consecutivePollFailures = 0;
758
- while (!isTerminalBackgroundStatus(readResponseStatus(payload))) {
914
+ while (useStoredBackgroundResponse &&
915
+ !isTerminalBackgroundStatus(readResponseStatus(payload))) {
759
916
  await new Promise((resolve) => setTimeout(resolve, BACKGROUND_POLL_INTERVAL_MS));
760
917
  try {
761
918
  const pollResponse = await fetch(buildResponsesUrl(profile, responseId), {
@@ -838,7 +995,7 @@ export class OpenAiResponsesProvider {
838
995
  }
839
996
  }
840
997
  const finalStatus = readResponseStatus(payload);
841
- if (finalStatus !== "completed") {
998
+ if (finalStatus && finalStatus !== "completed") {
842
999
  const errorMessage = readResponseError(payload) ??
843
1000
  `OpenAI background wiki compilation ended with status ${finalStatus}.`;
844
1001
  emitDiagnostic(logger, {
@@ -3,11 +3,10 @@ import { z } from "zod";
3
3
  import { getDatabase } from "./db.js";
4
4
  import { HttpError } from "./errors.js";
5
5
  import { recordActivityEvent } from "./repositories/activity-events.js";
6
- import { createNote, getNoteById, updateNote } from "./repositories/notes.js";
6
+ import { getNoteById, updateNote } from "./repositories/notes.js";
7
7
  import { createManualRewardGrant } from "./repositories/rewards.js";
8
8
  import { listTaskRuns } from "./repositories/task-runs.js";
9
9
  import { getDefaultUser } from "./repositories/users.js";
10
- import { listWikiSpaces } from "./repositories/wiki-memory.js";
11
10
  import { getScreenTimeOverlapSummary } from "./screen-time.js";
12
11
  const movementPublishModeSchema = z.enum([
13
12
  "auto_publish",
@@ -206,7 +205,7 @@ const movementTripInputSchema = z.object({
206
205
  });
207
206
  export const movementSettingsInputSchema = z.object({
208
207
  trackingEnabled: z.boolean().default(false),
209
- publishMode: movementPublishModeSchema.default("auto_publish"),
208
+ publishMode: movementPublishModeSchema.default("draft_review"),
210
209
  retentionMode: movementRetentionModeSchema.default("aggregates_only"),
211
210
  locationPermissionStatus: z.string().trim().default("not_determined"),
212
211
  motionPermissionStatus: z.string().trim().default("unknown"),
@@ -1030,7 +1029,7 @@ function defaultMovementSettings(userId) {
1030
1029
  return {
1031
1030
  userId,
1032
1031
  trackingEnabled: false,
1033
- publishMode: "auto_publish",
1032
+ publishMode: "draft_review",
1034
1033
  retentionMode: "aggregates_only",
1035
1034
  locationPermissionStatus: "not_determined",
1036
1035
  motionPermissionStatus: "unknown",
@@ -1278,7 +1277,7 @@ function ensureMovementSettings(userId) {
1278
1277
  background_tracking_ready, last_companion_sync_at, metadata_json,
1279
1278
  created_at, updated_at
1280
1279
  )
1281
- VALUES (?, 0, 'auto_publish', 'aggregates_only', 'not_determined', 'unknown', 0, NULL, '{}', ?, ?)`)
1280
+ VALUES (?, 0, 'draft_review', 'aggregates_only', 'not_determined', 'unknown', 0, NULL, '{}', ?, ?)`)
1282
1281
  .run(userId, now, now);
1283
1282
  return getMovementSettingsRow(userId);
1284
1283
  }
@@ -1363,9 +1362,6 @@ function listTripStops(tripIds) {
1363
1362
  ORDER BY trip_id ASC, sequence_index ASC`)
1364
1363
  .all(...tripIds);
1365
1364
  }
1366
- function defaultSpaceId() {
1367
- return listWikiSpaces()[0]?.id;
1368
- }
1369
1365
  function syncPlaceWikiMetadata(placeId) {
1370
1366
  const row = getDatabase()
1371
1367
  .prepare(`SELECT *
@@ -1561,176 +1557,6 @@ function resolvePlaceForPatch(input) {
1561
1557
  }
1562
1558
  return undefined;
1563
1559
  }
1564
- function createMovementNote(input) {
1565
- const spaceId = defaultSpaceId();
1566
- if (!spaceId) {
1567
- return null;
1568
- }
1569
- return createNote({
1570
- kind: "evidence",
1571
- title: input.title,
1572
- slug: "",
1573
- summary: "",
1574
- contentMarkdown: input.contentMarkdown,
1575
- spaceId,
1576
- parentSlug: null,
1577
- indexOrder: 0,
1578
- showInIndex: false,
1579
- aliases: [],
1580
- userId: input.userId,
1581
- author: null,
1582
- links: [],
1583
- tags: input.tags,
1584
- destroyAt: null,
1585
- sourcePath: "",
1586
- frontmatter: input.frontmatter,
1587
- revisionHash: ""
1588
- }, { actor: "Movement sync", source: "system" });
1589
- }
1590
- function formatMovementDurationForNote(valueSeconds) {
1591
- if (valueSeconds >= 86_400) {
1592
- return `${round(valueSeconds / 86_400, 1)} days`;
1593
- }
1594
- if (valueSeconds >= 3_600) {
1595
- return `${round(valueSeconds / 3_600, 1)} hours`;
1596
- }
1597
- return `${Math.max(1, Math.round(valueSeconds / 60))} minutes`;
1598
- }
1599
- function mergeMovementNoteTags(existingTags, existingFrontmatter, generatedTags) {
1600
- const movement = existingFrontmatter.movement &&
1601
- typeof existingFrontmatter.movement === "object" &&
1602
- !Array.isArray(existingFrontmatter.movement)
1603
- ? existingFrontmatter.movement
1604
- : null;
1605
- const previousGeneratedTags = Array.isArray(movement?.generatedTags)
1606
- ? movement.generatedTags.filter((value) => typeof value === "string")
1607
- : [];
1608
- const previousGeneratedTagSet = new Set(previousGeneratedTags.map((tag) => tag.toLowerCase()));
1609
- const preservedTags = existingTags.filter((tag) => !previousGeneratedTagSet.has(tag.toLowerCase()));
1610
- return uniqStrings([...preservedTags, ...generatedTags]);
1611
- }
1612
- function syncMovementNote(input) {
1613
- const existingNote = input.publishedNoteId
1614
- ? getNoteById(input.publishedNoteId)
1615
- : null;
1616
- if (existingNote && !Array.isArray(existingNote)) {
1617
- const updated = updateNote(existingNote.id, {
1618
- title: input.title,
1619
- contentMarkdown: input.contentMarkdown,
1620
- tags: mergeMovementNoteTags(existingNote.tags ?? [], existingNote.frontmatter, input.generatedTags),
1621
- frontmatter: {
1622
- ...existingNote.frontmatter,
1623
- ...input.frontmatter
1624
- }
1625
- }, { actor: "Movement sync", source: "system" });
1626
- return updated?.id ?? existingNote.id;
1627
- }
1628
- const created = createMovementNote({
1629
- userId: input.userId,
1630
- title: input.title,
1631
- contentMarkdown: input.contentMarkdown,
1632
- tags: input.generatedTags,
1633
- frontmatter: input.frontmatter
1634
- });
1635
- return created?.id ?? null;
1636
- }
1637
- function syncStayNote(settings, stay, place) {
1638
- if (!settings || settings.publishMode === "no_publish") {
1639
- return null;
1640
- }
1641
- const label = place?.label || stay.label || "Unlabeled stay";
1642
- const durationSecondsValue = durationSeconds(stay.started_at, stay.ended_at);
1643
- const live = stay.status.trim().toLowerCase() !== "completed" &&
1644
- stay.status.trim().toLowerCase() !== "closed";
1645
- const generatedTags = uniqStrings([
1646
- "movement",
1647
- "stay",
1648
- ...(place ? safeJsonParse(place.category_tags_json, []) : [])
1649
- ]);
1650
- const content = [
1651
- live ? `Currently staying at **${label}**.` : `Stayed at **${label}**.`,
1652
- "",
1653
- `- Started: ${stay.started_at}`,
1654
- `- ${live ? "Current end" : "Ended"}: ${stay.ended_at}`,
1655
- `- Duration: ${formatMovementDurationForNote(durationSecondsValue)}`,
1656
- `- Radius: ${Math.round(stay.radius_meters)} m`,
1657
- `- Classification: ${stay.classification || "stationary"}`
1658
- ].join("\n");
1659
- return syncMovementNote({
1660
- userId: stay.user_id,
1661
- publishedNoteId: stay.published_note_id,
1662
- title: `Stay · ${label}`,
1663
- contentMarkdown: content,
1664
- generatedTags,
1665
- frontmatter: {
1666
- observedAt: stay.started_at,
1667
- movement: {
1668
- kind: "stay",
1669
- state: live ? "live" : "closed",
1670
- stayId: stay.id,
1671
- publishMode: settings.publishMode,
1672
- placeId: place?.id ?? null,
1673
- placeLabel: label,
1674
- startedAt: stay.started_at,
1675
- endedAt: stay.ended_at,
1676
- durationSeconds: durationSecondsValue,
1677
- generatedTags
1678
- }
1679
- }
1680
- });
1681
- }
1682
- function syncTripNote(settings, trip, startPlace, endPlace) {
1683
- if (!settings || settings.publishMode === "no_publish") {
1684
- return null;
1685
- }
1686
- const startLabel = startPlace?.label || "Unknown start";
1687
- const endLabel = endPlace?.label || "Unknown end";
1688
- const durationSecondsValue = durationSeconds(trip.started_at, trip.ended_at);
1689
- const distanceKm = round(trip.distance_meters / 1000, 2);
1690
- const live = trip.status.trim().toLowerCase() !== "completed" &&
1691
- trip.status.trim().toLowerCase() !== "closed";
1692
- const generatedTags = uniqStrings([
1693
- "movement",
1694
- "trip",
1695
- ...safeJsonParse(trip.tags_json, [])
1696
- ]);
1697
- const content = [
1698
- live
1699
- ? `Currently moving from **${startLabel}** to **${endLabel}**.`
1700
- : `Travelled from **${startLabel}** to **${endLabel}**.`,
1701
- "",
1702
- `- Started: ${trip.started_at}`,
1703
- `- ${live ? "Current end" : "Ended"}: ${trip.ended_at}`,
1704
- `- Duration: ${formatMovementDurationForNote(durationSecondsValue)}`,
1705
- `- Distance: ${distanceKm} km`,
1706
- `- Activity: ${trip.activity_type || trip.travel_mode}`
1707
- ].join("\n");
1708
- return syncMovementNote({
1709
- userId: trip.user_id,
1710
- publishedNoteId: trip.published_note_id,
1711
- title: `Trip · ${startLabel} → ${endLabel}`,
1712
- contentMarkdown: content,
1713
- generatedTags,
1714
- frontmatter: {
1715
- observedAt: trip.started_at,
1716
- movement: {
1717
- kind: "trip",
1718
- state: live ? "live" : "closed",
1719
- tripId: trip.id,
1720
- publishMode: settings.publishMode,
1721
- startPlaceId: startPlace?.id ?? null,
1722
- endPlaceId: endPlace?.id ?? null,
1723
- startPlaceLabel: startLabel,
1724
- endPlaceLabel: endLabel,
1725
- startedAt: trip.started_at,
1726
- endedAt: trip.ended_at,
1727
- durationSeconds: durationSecondsValue,
1728
- distanceMeters: trip.distance_meters,
1729
- generatedTags
1730
- }
1731
- }
1732
- });
1733
- }
1734
1560
  function awardMovementXp(input) {
1735
1561
  const deltaXp = estimateMovementXp(input.categoryTags, input.distanceMeters);
1736
1562
  if (deltaXp <= 0) {
@@ -1905,16 +1731,6 @@ function upsertMovementStay(pairing, settings, input) {
1905
1731
  .prepare(`SELECT * FROM movement_stays WHERE user_id = ? AND external_uid = ?`)
1906
1732
  .get(pairing.user_id, parsed.externalUid);
1907
1733
  const freshMetadata = safeJsonParse(fresh.metadata_json, {});
1908
- if (settings?.publishMode === "auto_publish" && !hasInvalidMovementRecord(freshMetadata)) {
1909
- const publishedNoteId = syncStayNote(settings, fresh, matchedPlace);
1910
- if (publishedNoteId && publishedNoteId !== fresh.published_note_id) {
1911
- getDatabase()
1912
- .prepare(`UPDATE movement_stays
1913
- SET published_note_id = ?, updated_at = ?
1914
- WHERE id = ?`)
1915
- .run(publishedNoteId, nowIso(), fresh.id);
1916
- }
1917
- }
1918
1734
  return {
1919
1735
  mode: existing ? "updated" : "created",
1920
1736
  stayId: fresh.id
@@ -2040,16 +1856,6 @@ function upsertMovementTrip(pairing, settings, input) {
2040
1856
  .prepare(`SELECT * FROM movement_trips WHERE id = ?`)
2041
1857
  .get(fresh.id);
2042
1858
  const freshMetadata = safeJsonParse(refreshed.metadata_json, {});
2043
- if (settings?.publishMode === "auto_publish" && !hasInvalidMovementRecord(freshMetadata)) {
2044
- const publishedNoteId = syncTripNote(settings, refreshed, startPlace, endPlace);
2045
- if (publishedNoteId && publishedNoteId !== refreshed.published_note_id) {
2046
- getDatabase()
2047
- .prepare(`UPDATE movement_trips
2048
- SET published_note_id = ?, updated_at = ?
2049
- WHERE id = ?`)
2050
- .run(publishedNoteId, nowIso(), refreshed.id);
2051
- }
2052
- }
2053
1859
  if (!existing && settings?.publishMode === "auto_publish" && !hasInvalidMovementRecord(freshMetadata)) {
2054
1860
  awardMovementXp({
2055
1861
  userId: pairing.user_id,
@@ -10,6 +10,8 @@ import { createNoteLinkSchema, crudEntityTypeSchema, noteKindSchema, noteSchema
10
10
  import { deleteEncryptedSecret, readEncryptedSecret, storeEncryptedSecret } from "./calendar.js";
11
11
  import { isEntityDeleted } from "./deleted-entities.js";
12
12
  import { recordDiagnosticLog } from "./diagnostic-logs.js";
13
+ const MAX_WIKI_INGEST_TEXT_CHUNK_CHARS = 220_000;
14
+ const WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS = 2_000;
13
15
  const wikiSpaceSchema = z.object({
14
16
  id: z.string(),
15
17
  slug: z.string(),
@@ -1515,6 +1517,7 @@ export function listWikiPages(query) {
1515
1517
  ensureWikiSpaceSeedPages(spaceId);
1516
1518
  return listAllNotes()
1517
1519
  .filter((note) => note.spaceId === spaceId)
1520
+ .filter((note) => (query.includeHidden ? true : note.showInIndex))
1518
1521
  .filter((note) => (query.kind ? note.kind === query.kind : true))
1519
1522
  .sort(compareWikiPageOrder)
1520
1523
  .slice(0, query.limit ?? 100);
@@ -1638,6 +1641,7 @@ export async function searchWikiPages(input, secrets) {
1638
1641
  const parsed = wikiSearchQuerySchema.parse(input);
1639
1642
  const pages = listAllNotes()
1640
1643
  .filter((page) => (parsed.spaceId ? page.spaceId === parsed.spaceId : true))
1644
+ .filter((page) => page.showInIndex)
1641
1645
  .filter((page) => (parsed.kind ? page.kind === parsed.kind : true));
1642
1646
  const scores = new Map();
1643
1647
  const addScore = (noteId, value) => {
@@ -2015,6 +2019,115 @@ export async function reindexWikiEmbeddings(input, secrets) {
2015
2019
  chunkCount
2016
2020
  };
2017
2021
  }
2022
+ function isChunkableWikiIngestTextAsset(asset) {
2023
+ const mimeType = asset.mime_type.toLowerCase();
2024
+ const fileName = asset.file_name.toLowerCase();
2025
+ if (!existsSync(asset.file_path)) {
2026
+ return false;
2027
+ }
2028
+ if (asset.size_bytes <= MAX_WIKI_INGEST_TEXT_CHUNK_CHARS) {
2029
+ return false;
2030
+ }
2031
+ const metadata = parseJsonRecord(asset.metadata_json);
2032
+ if (metadata?.chunkParentAssetId || metadata?.textChunked) {
2033
+ return false;
2034
+ }
2035
+ return (mimeType.startsWith("text/") ||
2036
+ fileName.endsWith(".txt") ||
2037
+ fileName.endsWith(".md") ||
2038
+ fileName.endsWith(".markdown") ||
2039
+ fileName.endsWith(".csv") ||
2040
+ fileName.endsWith(".json"));
2041
+ }
2042
+ function splitWikiIngestTextIntoChunks(sourceText, maxChars) {
2043
+ const text = sourceText.trim();
2044
+ if (text.length <= maxChars) {
2045
+ return [text];
2046
+ }
2047
+ const chunks = [];
2048
+ let start = 0;
2049
+ while (start < text.length) {
2050
+ const hardEnd = Math.min(text.length, start + maxChars);
2051
+ let end = hardEnd;
2052
+ if (hardEnd < text.length) {
2053
+ const newline = text.lastIndexOf("\n", hardEnd);
2054
+ if (newline > start + Math.floor(maxChars * 0.65)) {
2055
+ end = newline + 1;
2056
+ }
2057
+ }
2058
+ const chunk = text.slice(start, end).trim();
2059
+ if (chunk.length > 0) {
2060
+ chunks.push(chunk);
2061
+ }
2062
+ if (end >= text.length) {
2063
+ break;
2064
+ }
2065
+ start = Math.max(0, end - WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS);
2066
+ }
2067
+ return chunks;
2068
+ }
2069
+ async function splitLargeWikiIngestTextAsset(options) {
2070
+ if (!isChunkableWikiIngestTextAsset(options.asset)) {
2071
+ return 0;
2072
+ }
2073
+ const sourceText = await readFile(options.asset.file_path, "utf8");
2074
+ const chunks = splitWikiIngestTextIntoChunks(sourceText, MAX_WIKI_INGEST_TEXT_CHUNK_CHARS);
2075
+ if (chunks.length <= 1) {
2076
+ return 0;
2077
+ }
2078
+ const extension = path.extname(options.asset.file_name) || ".txt";
2079
+ const baseName = path.basename(options.asset.file_name, extension) ||
2080
+ options.asset.file_name ||
2081
+ "source";
2082
+ const width = String(chunks.length).length;
2083
+ for (const [index, chunk] of chunks.entries()) {
2084
+ const chunkNumber = index + 1;
2085
+ const chunkFileName = `${baseName}-part-${String(chunkNumber).padStart(width, "0")}-of-${String(chunks.length).padStart(width, "0")}${extension}`;
2086
+ const chunkHeader = [
2087
+ `Source file: ${options.asset.file_name}`,
2088
+ `Source locator: ${options.asset.source_locator || options.asset.file_name}`,
2089
+ `Chunk: ${chunkNumber}/${chunks.length}`,
2090
+ `Parent checksum: ${options.asset.checksum}`,
2091
+ "",
2092
+ chunk
2093
+ ].join("\n");
2094
+ const persisted = await persistIngestUpload({
2095
+ jobId: options.jobId,
2096
+ fileName: chunkFileName,
2097
+ mimeType: options.asset.mime_type || "text/plain",
2098
+ payload: Buffer.from(chunkHeader, "utf8")
2099
+ });
2100
+ createWikiIngestAssetRecord({
2101
+ jobId: options.jobId,
2102
+ sourceKind: "upload",
2103
+ sourceLocator: `${options.asset.source_locator || options.asset.file_name}#chunk-${chunkNumber}`,
2104
+ fileName: chunkFileName,
2105
+ mimeType: options.asset.mime_type || "text/plain",
2106
+ filePath: persisted.filePath,
2107
+ sizeBytes: persisted.sizeBytes,
2108
+ checksum: persisted.checksum,
2109
+ metadata: {
2110
+ chunkParentAssetId: options.asset.id,
2111
+ chunkParentChecksum: options.asset.checksum,
2112
+ chunkIndex: chunkNumber,
2113
+ chunkCount: chunks.length,
2114
+ chunkOverlapChars: WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS,
2115
+ chunkMaxChars: MAX_WIKI_INGEST_TEXT_CHUNK_CHARS
2116
+ }
2117
+ });
2118
+ }
2119
+ updateWikiIngestAsset(options.asset.id, {
2120
+ status: "completed",
2121
+ metadata: {
2122
+ ...parseJsonRecord(options.asset.metadata_json),
2123
+ textChunked: true,
2124
+ textChunkCount: chunks.length,
2125
+ textChunkMaxChars: MAX_WIKI_INGEST_TEXT_CHUNK_CHARS,
2126
+ textChunkOverlapChars: WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS
2127
+ }
2128
+ });
2129
+ return chunks.length;
2130
+ }
2018
2131
  function getWikiIngestJobDir(jobId) {
2019
2132
  return path.join(resolveDataDir(), "wiki-ingest", jobId);
2020
2133
  }
@@ -2613,7 +2726,10 @@ export async function processWikiIngestJob(jobId, options) {
2613
2726
  const initialAssets = listWikiIngestJobAssetsInternal(jobId);
2614
2727
  let processedFiles = initialAssets.filter((asset) => asset.status === "completed").length;
2615
2728
  let totalFiles = Math.max(job.total_files, initialAssets.length);
2616
- let hadSuccess = false;
2729
+ let hadSuccess = (() => {
2730
+ const counts = refreshCounts();
2731
+ return counts.pageCount + counts.entityCount > 0;
2732
+ })();
2617
2733
  while (assetQueue().length > 0) {
2618
2734
  const nextAsset = assetQueue().find((asset) => ["processing", "queued"].includes(asset.status));
2619
2735
  if (!nextAsset) {
@@ -2675,6 +2791,27 @@ export async function processWikiIngestJob(jobId, options) {
2675
2791
  }
2676
2792
  continue;
2677
2793
  }
2794
+ const derivedChunkCount = await splitLargeWikiIngestTextAsset({
2795
+ jobId,
2796
+ asset: nextAsset
2797
+ });
2798
+ if (derivedChunkCount > 0) {
2799
+ totalFiles = Math.max(derivedChunkCount, totalFiles - 1 + derivedChunkCount);
2800
+ updateWikiIngestJob(jobId, {
2801
+ totalFiles,
2802
+ latestMessage: `Split ${nextAsset.file_name || "large text source"} into ${derivedChunkCount} chunks.`
2803
+ });
2804
+ createWikiIngestLog(jobId, `Split ${nextAsset.file_name || "large text source"} into ${derivedChunkCount} chunks.`, "info", {
2805
+ sourceAssetId: nextAsset.id,
2806
+ fileName: nextAsset.file_name,
2807
+ sourceLocator: nextAsset.source_locator,
2808
+ checksum: nextAsset.checksum,
2809
+ chunkCount: derivedChunkCount,
2810
+ chunkMaxChars: MAX_WIKI_INGEST_TEXT_CHUNK_CHARS,
2811
+ chunkOverlapChars: WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS
2812
+ });
2813
+ continue;
2814
+ }
2678
2815
  updateWikiIngestAsset(nextAsset.id, { status: "processing" });
2679
2816
  currentAssetContext = {
2680
2817
  assetId: nextAsset.id,
@@ -30,10 +30,13 @@ export async function buildCompanionPairingTransport(input) {
30
30
  pairPayload: snapshot.pairPayload,
31
31
  alpn: snapshot.alpn ?? COMPANION_IROH_ALPN,
32
32
  localBaseUrl: snapshot.localBaseUrl,
33
+ fallbackApiBaseUrl: requestApiBaseUrl,
34
+ fallbackUiBaseUrl: requestUiBaseUrl,
33
35
  recreateCommand: snapshot.recreateCommand ?? undefined,
34
36
  startedAt: snapshot.startedAt ?? undefined,
35
37
  notes: [
36
- "Default pairing uses Forge's Rust Iroh transport over QUIC.",
38
+ "Default pairing uses Forge's Rust Iroh transport over QUIC first.",
39
+ "The QR keeps the request API/UI URL as a direct fallback when Iroh cannot complete a request.",
37
40
  "The QR payload carries the Iroh node id, host token, optional relay, and ALPN forge-companion/1.",
38
41
  "Manual HTTP/TCP pairing remains available with --manual-http for advanced local setups."
39
42
  ]
@@ -200,12 +203,13 @@ function irohTransport(input) {
200
203
  const nodeId = input.pairPayload.node_id;
201
204
  return {
202
205
  transportMode: "iroh",
203
- apiBaseUrl: companionIrohApiBaseUrlFromNodeId(nodeId),
204
- uiBaseUrl: companionIrohUiBaseUrlFromNodeId(nodeId),
206
+ apiBaseUrl: input.fallbackApiBaseUrl,
207
+ uiBaseUrl: input.fallbackUiBaseUrl,
205
208
  transport: {
206
209
  protocol: "iroh",
207
210
  provider: "forge-companion-iroh",
208
211
  status: "ready",
212
+ publicBaseUrl: input.fallbackApiBaseUrl,
209
213
  localBaseUrl: input.localBaseUrl,
210
214
  nodeId,
211
215
  relay: input.pairPayload.relay,
@@ -297,6 +301,10 @@ function candidateIrohBinaries() {
297
301
  return candidateIrohAssetRoots().flatMap((root) => [
298
302
  path.join(root, "companion-iroh", "target", "release", binaryName),
299
303
  path.join(root, "companion-iroh", "target", "debug", binaryName),
304
+ path.join(root, "companion-iroh-src", "target", "release", binaryName),
305
+ path.join(root, "companion-iroh-src", "target", "debug", binaryName),
306
+ path.join(root, "dist", "companion-iroh-src", "target", "release", binaryName),
307
+ path.join(root, "dist", "companion-iroh-src", "target", "debug", binaryName),
300
308
  path.join(root, "openclaw-plugin", "dist", "companion-iroh", platformKey, binaryName),
301
309
  path.join(root, "companion-iroh", platformKey, binaryName),
302
310
  path.join(root, "companion-iroh", binaryName)
@@ -305,7 +313,8 @@ function candidateIrohBinaries() {
305
313
  function resolveCompanionIrohManifestPath() {
306
314
  const candidates = candidateIrohAssetRoots().flatMap((root) => [
307
315
  path.join(root, "companion-iroh", "Cargo.toml"),
308
- path.join(root, "companion-iroh-src", "Cargo.toml")
316
+ path.join(root, "companion-iroh-src", "Cargo.toml"),
317
+ path.join(root, "dist", "companion-iroh-src", "Cargo.toml")
309
318
  ]);
310
319
  return candidates.find((candidate) => existsSync(candidate)) ?? null;
311
320
  }
@@ -2,7 +2,7 @@
2
2
  "id": "forge-openclaw-plugin",
3
3
  "name": "Forge",
4
4
  "description": "Curated OpenClaw adapter for the Forge collaboration API, UI entrypoint, and localhost auto-start runtime.",
5
- "version": "0.2.114",
5
+ "version": "0.2.116",
6
6
  "activation": {
7
7
  "onStartup": true,
8
8
  "onCapabilities": [
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "forge-openclaw-plugin",
3
- "version": "0.2.114",
3
+ "version": "0.2.116",
4
4
  "description": "Curated OpenClaw adapter for the Forge collaboration API, UI entrypoint, and localhost auto-start runtime.",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -0,0 +1,12 @@
1
+ UPDATE movement_stays
2
+ SET published_note_id = NULL
3
+ WHERE published_note_id IS NOT NULL;
4
+
5
+ UPDATE movement_trips
6
+ SET published_note_id = NULL
7
+ WHERE published_note_id IS NOT NULL;
8
+
9
+ DELETE FROM notes
10
+ WHERE source = 'system'
11
+ AND kind = 'evidence'
12
+ AND json_extract(frontmatter_json, '$.movement.kind') IN ('stay', 'trip');