forge-openclaw-plugin 0.2.114 → 0.2.116
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/openclaw/local-runtime.js +86 -1
- package/dist/server/server/migrations/068_remove_generated_movement_wiki_logs.sql +12 -0
- package/dist/server/server/src/app.js +2 -1
- package/dist/server/server/src/discovery-advertiser.js +3 -7
- package/dist/server/server/src/managers/platform/openai-responses-provider.js +192 -35
- package/dist/server/server/src/movement.js +4 -198
- package/dist/server/server/src/repositories/wiki-memory.js +138 -1
- package/dist/server/server/src/services/companion-iroh.js +13 -4
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/server/migrations/068_remove_generated_movement_wiki_logs.sql +12 -0
- package/dist/companion-iroh/darwin-arm64/forge-companion-iroh +0 -0
- package/dist/companion-iroh/darwin-x64/forge-companion-iroh +0 -0
- package/dist/companion-iroh/linux-x64/forge-companion-iroh +0 -0
- package/dist/companion-iroh/win32-x64/forge-companion-iroh.exe +0 -0
|
@@ -3,7 +3,7 @@ import { closeSync, existsSync, mkdirSync, openSync } from "node:fs";
|
|
|
3
3
|
import net from "node:net";
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
5
|
import path from "node:path";
|
|
6
|
-
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
6
|
+
import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
8
|
const LOCAL_HOSTNAMES = new Set(["127.0.0.1", "localhost", "::1"]);
|
|
9
9
|
const STARTUP_TIMEOUT_MS = 15_000;
|
|
@@ -37,6 +37,12 @@ function getRuntimeStatePath(config) {
|
|
|
37
37
|
const origin = new URL(config.origin).hostname.toLowerCase().replace(/[^a-z0-9._-]+/g, "-");
|
|
38
38
|
return path.join(homedir(), ".openclaw", "run", FORGE_PLUGIN_ID, `${origin}-${config.port}.json`);
|
|
39
39
|
}
|
|
40
|
+
function getRuntimeStateDir() {
|
|
41
|
+
return path.join(homedir(), ".openclaw", "run", FORGE_PLUGIN_ID);
|
|
42
|
+
}
|
|
43
|
+
function getRuntimeStateOrigin(config) {
|
|
44
|
+
return new URL(config.origin).hostname.toLowerCase().replace(/[^a-z0-9._-]+/g, "-");
|
|
45
|
+
}
|
|
40
46
|
function getPreferredPortStatePath(origin) {
|
|
41
47
|
const hostname = new URL(origin).hostname.toLowerCase().replace(/[^a-z0-9._-]+/g, "-");
|
|
42
48
|
return path.join(homedir(), ".openclaw", "run", FORGE_PLUGIN_ID, `${hostname}-preferred-port.json`);
|
|
@@ -118,6 +124,10 @@ async function writeRuntimeState(config, pid) {
|
|
|
118
124
|
async function clearRuntimeState(config) {
|
|
119
125
|
await rm(getRuntimeStatePath(config), { force: true });
|
|
120
126
|
}
|
|
127
|
+
async function clearRuntimeStateForState(state) {
|
|
128
|
+
const origin = new URL(state.origin).hostname.toLowerCase().replace(/[^a-z0-9._-]+/g, "-");
|
|
129
|
+
await rm(path.join(getRuntimeStateDir(), `${origin}-${state.port}.json`), { force: true });
|
|
130
|
+
}
|
|
121
131
|
async function readRuntimeState(config) {
|
|
122
132
|
try {
|
|
123
133
|
const payload = await readFile(getRuntimeStatePath(config), "utf8");
|
|
@@ -138,6 +148,52 @@ async function readRuntimeState(config) {
|
|
|
138
148
|
return null;
|
|
139
149
|
}
|
|
140
150
|
}
|
|
151
|
+
async function readRuntimeStateFile(filePath) {
|
|
152
|
+
try {
|
|
153
|
+
const payload = await readFile(filePath, "utf8");
|
|
154
|
+
const parsed = JSON.parse(payload);
|
|
155
|
+
if (typeof parsed.pid !== "number" ||
|
|
156
|
+
!Number.isFinite(parsed.pid) ||
|
|
157
|
+
typeof parsed.origin !== "string" ||
|
|
158
|
+
typeof parsed.port !== "number" ||
|
|
159
|
+
!Number.isFinite(parsed.port)) {
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
return {
|
|
163
|
+
pid: Math.trunc(parsed.pid),
|
|
164
|
+
origin: parsed.origin,
|
|
165
|
+
port: Math.trunc(parsed.port),
|
|
166
|
+
baseUrl: typeof parsed.baseUrl === "string" ? parsed.baseUrl : buildForgeBaseUrl(parsed.origin, parsed.port),
|
|
167
|
+
startedAt: typeof parsed.startedAt === "string" ? parsed.startedAt : new Date(0).toISOString(),
|
|
168
|
+
logPath: typeof parsed.logPath === "string" ? parsed.logPath : null
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
return null;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
async function readRuntimeStatesForOrigin(config) {
|
|
176
|
+
const stateDir = getRuntimeStateDir();
|
|
177
|
+
const origin = getRuntimeStateOrigin(config);
|
|
178
|
+
let entries;
|
|
179
|
+
try {
|
|
180
|
+
entries = await readdir(stateDir);
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
return [];
|
|
184
|
+
}
|
|
185
|
+
const states = [];
|
|
186
|
+
for (const entry of entries) {
|
|
187
|
+
if (!entry.startsWith(`${origin}-`) || !entry.endsWith(".json") || entry.endsWith("-preferred-port.json")) {
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
const state = await readRuntimeStateFile(path.join(stateDir, entry));
|
|
191
|
+
if (state) {
|
|
192
|
+
states.push(state);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return states;
|
|
196
|
+
}
|
|
141
197
|
function processExists(pid) {
|
|
142
198
|
try {
|
|
143
199
|
process.kill(pid, 0);
|
|
@@ -147,6 +203,34 @@ function processExists(pid) {
|
|
|
147
203
|
return !(error instanceof Error) || !("code" in error) || error.code !== "ESRCH";
|
|
148
204
|
}
|
|
149
205
|
}
|
|
206
|
+
async function cleanupSupersededManagedRuntimes(config, expectedDataRoot) {
|
|
207
|
+
const states = await readRuntimeStatesForOrigin(config);
|
|
208
|
+
for (const state of states) {
|
|
209
|
+
if (state.port === config.port) {
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
if (!processExists(state.pid)) {
|
|
213
|
+
await clearRuntimeStateForState(state);
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
const alternateConfig = {
|
|
217
|
+
...config,
|
|
218
|
+
port: state.port,
|
|
219
|
+
baseUrl: buildForgeBaseUrl(state.origin, state.port),
|
|
220
|
+
webAppUrl: buildForgeWebAppUrl(state.origin, state.port)
|
|
221
|
+
};
|
|
222
|
+
const alternateProbe = await probeForgeRuntime(alternateConfig, HEALTHCHECK_TIMEOUT_MS);
|
|
223
|
+
if (!alternateProbe.healthy || !isExpectedDataRoot(expectedDataRoot, alternateProbe.storageRoot)) {
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
process.kill(state.pid, "SIGTERM");
|
|
227
|
+
if (!(await waitForProcessExit(state.pid, 5_000))) {
|
|
228
|
+
process.kill(state.pid, "SIGKILL");
|
|
229
|
+
await waitForProcessExit(state.pid, 2_000);
|
|
230
|
+
}
|
|
231
|
+
await clearRuntimeStateForState(state);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
150
234
|
async function waitForProcessExit(pid, timeoutMs) {
|
|
151
235
|
const deadline = Date.now() + timeoutMs;
|
|
152
236
|
while (Date.now() < deadline) {
|
|
@@ -490,6 +574,7 @@ export async function ensureForgeRuntimeReady(config) {
|
|
|
490
574
|
const expectedDataRoot = getExpectedDataRoot(config);
|
|
491
575
|
const initialProbe = await probeForgeRuntime(config, HEALTHCHECK_TIMEOUT_MS);
|
|
492
576
|
if (initialProbe.healthy && isExpectedDataRoot(expectedDataRoot, initialProbe.storageRoot)) {
|
|
577
|
+
await cleanupSupersededManagedRuntimes(config, expectedDataRoot);
|
|
493
578
|
const existingState = await readRuntimeState(config);
|
|
494
579
|
if (!existingState) {
|
|
495
580
|
await adoptManagedRuntimeState(config, initialProbe);
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
UPDATE movement_stays
|
|
2
|
+
SET published_note_id = NULL
|
|
3
|
+
WHERE published_note_id IS NOT NULL;
|
|
4
|
+
|
|
5
|
+
UPDATE movement_trips
|
|
6
|
+
SET published_note_id = NULL
|
|
7
|
+
WHERE published_note_id IS NOT NULL;
|
|
8
|
+
|
|
9
|
+
DELETE FROM notes
|
|
10
|
+
WHERE source = 'system'
|
|
11
|
+
AND kind = 'evidence'
|
|
12
|
+
AND json_extract(frontmatter_json, '$.movement.kind') IN ('stay', 'trip');
|
|
@@ -9179,7 +9179,8 @@ export async function buildServer(options = {}) {
|
|
|
9179
9179
|
pages: listWikiPages({
|
|
9180
9180
|
spaceId: query.spaceId,
|
|
9181
9181
|
kind: query.kind,
|
|
9182
|
-
limit: query.limit ? Number(query.limit) : undefined
|
|
9182
|
+
limit: query.limit ? Number(query.limit) : undefined,
|
|
9183
|
+
includeHidden: query.includeHidden === "true"
|
|
9183
9184
|
})
|
|
9184
9185
|
};
|
|
9185
9186
|
});
|
|
@@ -3,7 +3,7 @@ import os from "node:os";
|
|
|
3
3
|
import { promisify } from "node:util";
|
|
4
4
|
import bonjourService from "bonjour-service";
|
|
5
5
|
import { logForgeDebug } from "./debug.js";
|
|
6
|
-
import {
|
|
6
|
+
import { getCompanionIrohStatus } from "./services/companion-iroh.js";
|
|
7
7
|
const execFileAsync = promisify(execFile);
|
|
8
8
|
const BonjourConstructor = bonjourService.Bonjour ??
|
|
9
9
|
bonjourService.default ??
|
|
@@ -37,12 +37,8 @@ export async function startForgeDiscoveryAdvertiser(options) {
|
|
|
37
37
|
tsApiBaseUrl: tailscaleTargets.apiBaseUrl ?? "",
|
|
38
38
|
tsUiBaseUrl: tailscaleTargets.uiBaseUrl ?? "",
|
|
39
39
|
tsDnsName: tailscaleTargets.dnsName ?? "",
|
|
40
|
-
irohApiBaseUrl: irohNodeId
|
|
41
|
-
|
|
42
|
-
: "",
|
|
43
|
-
irohUiBaseUrl: irohNodeId
|
|
44
|
-
? companionIrohUiBaseUrlFromNodeId(irohNodeId)
|
|
45
|
-
: "",
|
|
40
|
+
irohApiBaseUrl: irohNodeId ? tailscaleTargets.apiBaseUrl ?? "" : "",
|
|
41
|
+
irohUiBaseUrl: irohNodeId ? tailscaleTargets.uiBaseUrl ?? "" : "",
|
|
46
42
|
irohProvider: irohNodeId ? "forge-companion-iroh" : "",
|
|
47
43
|
irohNodeId: irohNodeId ?? "",
|
|
48
44
|
irohRelay: irohTransport.pairPayload?.relay ?? "",
|
|
@@ -20,8 +20,11 @@ const MODEL_CONTEXT_WINDOWS = {
|
|
|
20
20
|
};
|
|
21
21
|
const DEFAULT_CONTEXT_WINDOW = 400_000;
|
|
22
22
|
const RESERVED_RESPONSE_TOKENS = 140_000;
|
|
23
|
+
const CODEX_WIKI_COMPILE_CONTEXT_WINDOW = 120_000;
|
|
24
|
+
const CODEX_WIKI_COMPILE_RESERVED_RESPONSE_TOKENS = 60_000;
|
|
23
25
|
const APPROX_CHARS_PER_TOKEN = 4;
|
|
24
26
|
const REQUEST_TIMEOUT_MS = 90_000;
|
|
27
|
+
const CODEX_FOREGROUND_COMPILE_TIMEOUT_MS = 10 * 60_000;
|
|
25
28
|
const BACKGROUND_POLL_INTERVAL_MS = 2_000;
|
|
26
29
|
const DEFAULT_CODEX_BASE_URL = "https://chatgpt.com/backend-api";
|
|
27
30
|
const CODEX_JWT_CLAIM_PATH = "https://api.openai.com/auth";
|
|
@@ -164,6 +167,106 @@ function parseOutputText(payload) {
|
|
|
164
167
|
}
|
|
165
168
|
return null;
|
|
166
169
|
}
|
|
170
|
+
function buildOutputTextPayload(text) {
|
|
171
|
+
return {
|
|
172
|
+
status: "completed",
|
|
173
|
+
output: [
|
|
174
|
+
{
|
|
175
|
+
content: [{ type: "output_text", text }]
|
|
176
|
+
}
|
|
177
|
+
]
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
function parseCodexEventStreamPayload(streamText) {
|
|
181
|
+
const chunks = [];
|
|
182
|
+
let latestResponse = null;
|
|
183
|
+
let failedError = null;
|
|
184
|
+
let dataLines = [];
|
|
185
|
+
const flushEvent = () => {
|
|
186
|
+
if (dataLines.length === 0) {
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
const raw = dataLines.join("\n");
|
|
190
|
+
dataLines = [];
|
|
191
|
+
if (raw.trim() === "[DONE]") {
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
let payload;
|
|
195
|
+
try {
|
|
196
|
+
payload = JSON.parse(raw);
|
|
197
|
+
}
|
|
198
|
+
catch {
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
const payloadType = typeof payload.type === "string" ? payload.type : null;
|
|
202
|
+
if (payloadType === "response.output_text.delta") {
|
|
203
|
+
if (typeof payload.delta === "string") {
|
|
204
|
+
chunks.push(payload.delta);
|
|
205
|
+
}
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if (payloadType === "response.output_text.done") {
|
|
209
|
+
if (typeof payload.text === "string" && chunks.length === 0) {
|
|
210
|
+
chunks.push(payload.text);
|
|
211
|
+
}
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
if (payloadType === "response.completed" ||
|
|
215
|
+
payloadType === "response.failed") {
|
|
216
|
+
const response = payload.response;
|
|
217
|
+
if (response && typeof response === "object") {
|
|
218
|
+
latestResponse = response;
|
|
219
|
+
const text = parseOutputText(latestResponse);
|
|
220
|
+
if (text && chunks.length === 0) {
|
|
221
|
+
chunks.push(text);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
if (payloadType === "response.failed") {
|
|
225
|
+
failedError =
|
|
226
|
+
payload.error ??
|
|
227
|
+
latestResponse?.error ??
|
|
228
|
+
"Codex response failed.";
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
};
|
|
232
|
+
for (const line of streamText.split(/\r?\n/)) {
|
|
233
|
+
if (line === "") {
|
|
234
|
+
flushEvent();
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
237
|
+
if (line.startsWith(":")) {
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
if (line.startsWith("data:")) {
|
|
241
|
+
dataLines.push(line.slice("data:".length).trimStart());
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
flushEvent();
|
|
245
|
+
if (failedError) {
|
|
246
|
+
throw new Error(`Codex response failed: ${JSON.stringify(failedError)}`);
|
|
247
|
+
}
|
|
248
|
+
const text = chunks.join("");
|
|
249
|
+
if (latestResponse) {
|
|
250
|
+
if (!parseOutputText(latestResponse) && text) {
|
|
251
|
+
return buildOutputTextPayload(text);
|
|
252
|
+
}
|
|
253
|
+
return latestResponse;
|
|
254
|
+
}
|
|
255
|
+
if (text) {
|
|
256
|
+
return buildOutputTextPayload(text);
|
|
257
|
+
}
|
|
258
|
+
try {
|
|
259
|
+
return JSON.parse(streamText);
|
|
260
|
+
}
|
|
261
|
+
catch {
|
|
262
|
+
return buildOutputTextPayload("");
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
async function readProviderPayload(response, profile) {
|
|
266
|
+
return isCodexProfile(profile)
|
|
267
|
+
? parseCodexEventStreamPayload(await response.text())
|
|
268
|
+
: readJsonPayload(response);
|
|
269
|
+
}
|
|
167
270
|
function readReasoningEffort(profile) {
|
|
168
271
|
return typeof profile.metadata.reasoningEffort === "string"
|
|
169
272
|
? profile.metadata.reasoningEffort
|
|
@@ -233,6 +336,7 @@ function buildRequestHeaders(profile, apiKey, options = {}) {
|
|
|
233
336
|
headers["OpenAI-Beta"] = "responses=experimental";
|
|
234
337
|
headers.originator = "pi";
|
|
235
338
|
headers["chatgpt-account-id"] = extractCodexAccountId(apiKey);
|
|
339
|
+
headers.accept = "text/event-stream";
|
|
236
340
|
return headers;
|
|
237
341
|
}
|
|
238
342
|
function buildReasoningConfiguration(profile) {
|
|
@@ -250,12 +354,21 @@ function buildTextConfiguration(options) {
|
|
|
250
354
|
}
|
|
251
355
|
return Object.keys(text).length > 0 ? text : undefined;
|
|
252
356
|
}
|
|
357
|
+
function buildInstructionsPayload(profile, instructions) {
|
|
358
|
+
return isCodexProfile(profile) ? { instructions } : {};
|
|
359
|
+
}
|
|
253
360
|
function estimateTokens(text) {
|
|
254
361
|
return Math.ceil(text.length / APPROX_CHARS_PER_TOKEN);
|
|
255
362
|
}
|
|
256
363
|
function computeSourceExcerpt(profile, sourceText) {
|
|
257
|
-
const
|
|
258
|
-
const
|
|
364
|
+
const configuredContextWindow = MODEL_CONTEXT_WINDOWS[profile.model] ?? DEFAULT_CONTEXT_WINDOW;
|
|
365
|
+
const contextWindow = isCodexProfile(profile)
|
|
366
|
+
? Math.min(configuredContextWindow, CODEX_WIKI_COMPILE_CONTEXT_WINDOW)
|
|
367
|
+
: configuredContextWindow;
|
|
368
|
+
const reservedResponseTokens = isCodexProfile(profile)
|
|
369
|
+
? CODEX_WIKI_COMPILE_RESERVED_RESPONSE_TOKENS
|
|
370
|
+
: RESERVED_RESPONSE_TOKENS;
|
|
371
|
+
const inputBudget = Math.max(16_000, contextWindow - reservedResponseTokens);
|
|
259
372
|
const estimatedTokens = estimateTokens(sourceText);
|
|
260
373
|
if (estimatedTokens <= inputBudget) {
|
|
261
374
|
return {
|
|
@@ -359,7 +472,11 @@ export class OpenAiResponsesProvider {
|
|
|
359
472
|
}),
|
|
360
473
|
body: JSON.stringify({
|
|
361
474
|
model: profile.model,
|
|
362
|
-
|
|
475
|
+
...buildInstructionsPayload(profile, "Reply with the single word ok."),
|
|
476
|
+
input: isCodexProfile(profile)
|
|
477
|
+
? "Connection test."
|
|
478
|
+
: "Reply with the single word ok.",
|
|
479
|
+
...(isCodexProfile(profile) ? { stream: true, store: false } : {}),
|
|
363
480
|
max_output_tokens: 24,
|
|
364
481
|
reasoning: buildReasoningConfiguration(profile),
|
|
365
482
|
text: buildTextConfiguration({ profile })
|
|
@@ -404,7 +521,7 @@ export class OpenAiResponsesProvider {
|
|
|
404
521
|
});
|
|
405
522
|
throw new Error(`OpenAI connection test failed (${response.status})${message ? `: ${message}` : ""}`);
|
|
406
523
|
}
|
|
407
|
-
const payload = await
|
|
524
|
+
const payload = await readProviderPayload(response, profile);
|
|
408
525
|
emitDiagnostic(logger, {
|
|
409
526
|
level: "info",
|
|
410
527
|
message: "OpenAI connection test completed.",
|
|
@@ -440,20 +557,31 @@ export class OpenAiResponsesProvider {
|
|
|
440
557
|
}),
|
|
441
558
|
body: JSON.stringify({
|
|
442
559
|
model: profile.model,
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
560
|
+
...buildInstructionsPayload(profile, systemPrompt?.trim() || "Follow the user's request."),
|
|
561
|
+
input: isCodexProfile(profile)
|
|
562
|
+
? [
|
|
563
|
+
{
|
|
564
|
+
role: "user",
|
|
565
|
+
content: [{ type: "input_text", text: prompt }]
|
|
566
|
+
}
|
|
567
|
+
]
|
|
568
|
+
: [
|
|
569
|
+
...(systemPrompt?.trim()
|
|
570
|
+
? [
|
|
571
|
+
{
|
|
572
|
+
role: "system",
|
|
573
|
+
content: [
|
|
574
|
+
{ type: "input_text", text: systemPrompt.trim() }
|
|
575
|
+
]
|
|
576
|
+
}
|
|
577
|
+
]
|
|
578
|
+
: []),
|
|
579
|
+
{
|
|
580
|
+
role: "user",
|
|
581
|
+
content: [{ type: "input_text", text: prompt }]
|
|
582
|
+
}
|
|
583
|
+
],
|
|
584
|
+
...(isCodexProfile(profile) ? { stream: true, store: false } : {}),
|
|
457
585
|
reasoning: buildReasoningConfiguration(profile),
|
|
458
586
|
text: buildTextConfiguration({ profile }),
|
|
459
587
|
max_output_tokens: 1200
|
|
@@ -463,7 +591,7 @@ export class OpenAiResponsesProvider {
|
|
|
463
591
|
const message = await response.text();
|
|
464
592
|
throw new Error(`OpenAI text prompt failed (${response.status})${message ? `: ${message}` : ""}`);
|
|
465
593
|
}
|
|
466
|
-
const payload = await
|
|
594
|
+
const payload = await readProviderPayload(response, profile);
|
|
467
595
|
return {
|
|
468
596
|
outputText: parseOutputText(payload)?.trim() || ""
|
|
469
597
|
};
|
|
@@ -477,16 +605,16 @@ export class OpenAiResponsesProvider {
|
|
|
477
605
|
"Goal:",
|
|
478
606
|
"- Produce one main overview page in markdown.",
|
|
479
607
|
"- Split durable subtopics into articleCandidates with full draft markdown.",
|
|
480
|
-
"- Propose
|
|
608
|
+
"- Propose structured operational entities only when the source clearly supports a durable record.",
|
|
481
609
|
"- Suggest page updates only when the source clearly belongs in an existing page instead of a new page.",
|
|
482
610
|
"What Forge expects:",
|
|
483
611
|
"- The main markdown should be a readable overview or anchor page, not a raw source dump.",
|
|
484
612
|
"- articleCandidates should contain real draft wiki pages with their own markdown, not just titles.",
|
|
485
613
|
"- entityProposals must use one of these entityType values only: goal, project, task, habit, strategy, psyche_value, note.",
|
|
486
614
|
"- Use suggestedFields only for fields that truly fit the entity type. Use null for unknown scalar fields and [] for unknown list fields.",
|
|
487
|
-
"- Keep entity proposals conservative. Prefer wiki
|
|
615
|
+
"- Keep structured operational entity proposals conservative. Prefer wiki page entities when the source is informative but not actionable enough for goals, projects, tasks, habits, strategies, values, or evidence notes.",
|
|
488
616
|
"Forge ontology:",
|
|
489
|
-
"- Forge has two durable
|
|
617
|
+
"- Forge has two durable entity families: wiki page entities and structured operational entities.",
|
|
490
618
|
"- Use wiki pages for rich context, summaries, explanations, relationships, timelines, source synthesis, and themes that are broader than one action item.",
|
|
491
619
|
"- Use entities for operational objects that Forge can track directly: goals, projects, tasks, habits, strategies, psyche values, and durable notes.",
|
|
492
620
|
"- When the same topic needs both explanation and operations, create both: a wiki page for context and an entity proposal for the operational record.",
|
|
@@ -498,11 +626,25 @@ export class OpenAiResponsesProvider {
|
|
|
498
626
|
"- For chats and transcripts, extract the durable parts: people, relationships, ongoing projects, commitments, habits, values, decisions, questions, sources, and evidence.",
|
|
499
627
|
"- Merge repetitive back-and-forth into concise summaries.",
|
|
500
628
|
"- Use short quotes only when the exact phrase matters.",
|
|
629
|
+
"Sensitive information rules:",
|
|
630
|
+
"- Never store or reproduce secrets, passwords, passphrases, recovery codes, API keys, access tokens, refresh tokens, session cookies, private keys, seed phrases, one-time codes, full payment card numbers, or equivalent credentials.",
|
|
631
|
+
"- If the source contains a secret or credential, replace the value with [REDACTED SECRET] and keep only the minimum non-secret context needed to explain why it appeared.",
|
|
632
|
+
"- Do not create wiki pages, notes, entity proposals, aliases, tags, titles, or page update suggestions that preserve secret values.",
|
|
501
633
|
"How to split pages:",
|
|
502
634
|
"- Keep markdown as the overview page for this source.",
|
|
503
635
|
"- If one topic deserves its own page, put it in articleCandidates with title, slug, summary, rationale, markdown, tags, aliases, and parentSlug.",
|
|
504
636
|
"- Use parentSlug when the draft page clearly belongs under an existing Forge wiki branch such as people, projects, concepts, sources, or chronicle.",
|
|
637
|
+
"- For chats and message logs, create articleCandidates for durable people, family members, partners, collaborators, sources, places, projects, important dated events, recurring themes, and concepts when the source contains enough reusable context to make a useful page.",
|
|
638
|
+
"- Do not leave durable people only as [[links]] when the source gives relationship, role, timeline, or follow-up context; create a lightweight person page candidate instead.",
|
|
639
|
+
"- Do not leave important or specific events only as bullets in the overview when they have their own timeline, decision, consequence, open loop, or later follow-up value; create a chronicle page candidate instead.",
|
|
640
|
+
"- Do not leave recurring concepts only as inline observations when the source gives definitions, examples, tradeoffs, or repeated behavior patterns; create a concept page candidate instead.",
|
|
505
641
|
"- Do not create articleCandidates for every minor mention; only create pages that would be useful to reopen later.",
|
|
642
|
+
"Coverage audit before output:",
|
|
643
|
+
"- Before final JSON, scan the source for page-worthy entities in these categories: people, family/partners, organizations, places, sources/files/links, dated events, decisions, recurring concepts, health/care episodes, projects/plans, and relationship contexts.",
|
|
644
|
+
"- For each category with enough durable context, either create an articleCandidate or make the reason for not splitting clear inside the overview page.",
|
|
645
|
+
"- For a close relationship chat, it is normal to create several page candidates: the source/export page, the main person page, close family/partner pages when named with context, and major event pages such as meetups, care crises, travel episodes, or important decisions.",
|
|
646
|
+
"- Prefer fewer high-quality pages over many stubs, but do not collapse distinct durable people or major events into one page merely because they came from the same file.",
|
|
647
|
+
"- If the source has a named spouse/partner, family member, collaborator, or recurring person with role/context/follow-up value, create a person page candidate even if the page is short.",
|
|
506
648
|
"Useful high-level wiki themes:",
|
|
507
649
|
"- people: people, collaborators, family, teams, roles, relationship context.",
|
|
508
650
|
"- projects: bounded initiatives, active efforts, plans, milestones, workstreams.",
|
|
@@ -528,7 +670,7 @@ export class OpenAiResponsesProvider {
|
|
|
528
670
|
"Entity proposal rules:",
|
|
529
671
|
"- Never invent IDs.",
|
|
530
672
|
"- Do not propose an entity just because it is mentioned once without durable importance.",
|
|
531
|
-
"- People, concepts, sources, places, and broad life areas are usually wiki
|
|
673
|
+
"- People, concepts, sources, places, and broad life areas are usually wiki page entities, not structured operational entity proposals.",
|
|
532
674
|
"- Goals should be outcomes, not chores.",
|
|
533
675
|
"- Projects should group multiple steps or phases, not a single errand.",
|
|
534
676
|
"- Tasks should be concrete and actionable enough to do soon.",
|
|
@@ -617,6 +759,8 @@ export class OpenAiResponsesProvider {
|
|
|
617
759
|
role: "user",
|
|
618
760
|
content: userContent
|
|
619
761
|
});
|
|
762
|
+
const requestInputs = isCodexProfile(profile) ? inputs.slice(1) : inputs;
|
|
763
|
+
const useStoredBackgroundResponse = !isCodexProfile(profile);
|
|
620
764
|
let payload;
|
|
621
765
|
let responseId = resumeResponseId?.trim() || null;
|
|
622
766
|
if (responseId) {
|
|
@@ -667,11 +811,17 @@ export class OpenAiResponsesProvider {
|
|
|
667
811
|
}),
|
|
668
812
|
body: JSON.stringify({
|
|
669
813
|
model: profile.model,
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
814
|
+
...buildInstructionsPayload(profile, prompt),
|
|
815
|
+
input: requestInputs,
|
|
816
|
+
store: useStoredBackgroundResponse,
|
|
817
|
+
...(isCodexProfile(profile) ? { stream: true } : {}),
|
|
818
|
+
...(useStoredBackgroundResponse
|
|
819
|
+
? {
|
|
820
|
+
background: true,
|
|
821
|
+
prompt_cache_retention: profile.model === "gpt-5.4" ? "24h" : "in_memory",
|
|
822
|
+
prompt_cache_key: `forge-wiki-ingest:${profile.model}:${input.parseStrategy}:${input.mimeType}`
|
|
823
|
+
}
|
|
824
|
+
: {}),
|
|
675
825
|
reasoning: buildReasoningConfiguration(profile),
|
|
676
826
|
text: buildTextConfiguration({
|
|
677
827
|
profile,
|
|
@@ -683,7 +833,9 @@ export class OpenAiResponsesProvider {
|
|
|
683
833
|
}
|
|
684
834
|
})
|
|
685
835
|
}),
|
|
686
|
-
signal: AbortSignal.timeout(
|
|
836
|
+
signal: AbortSignal.timeout(isCodexProfile(profile)
|
|
837
|
+
? CODEX_FOREGROUND_COMPILE_TIMEOUT_MS
|
|
838
|
+
: REQUEST_TIMEOUT_MS)
|
|
687
839
|
});
|
|
688
840
|
}
|
|
689
841
|
catch (error) {
|
|
@@ -733,17 +885,21 @@ export class OpenAiResponsesProvider {
|
|
|
733
885
|
});
|
|
734
886
|
throw new Error(`LLM compilation failed: ${createResponse.status}${message ? `: ${message}` : ""}`);
|
|
735
887
|
}
|
|
736
|
-
payload = await
|
|
888
|
+
payload = await readProviderPayload(createResponse, profile);
|
|
737
889
|
responseId = readResponseId(payload);
|
|
738
|
-
if (!responseId) {
|
|
890
|
+
if (useStoredBackgroundResponse && !responseId) {
|
|
739
891
|
throw new Error("OpenAI background response did not include an id for polling.");
|
|
740
892
|
}
|
|
741
893
|
emitDiagnostic(logger, {
|
|
742
894
|
level: "info",
|
|
743
|
-
message:
|
|
895
|
+
message: useStoredBackgroundResponse
|
|
896
|
+
? "OpenAI accepted the wiki compilation job for background processing."
|
|
897
|
+
: "OpenAI Codex returned a foreground wiki compilation response.",
|
|
744
898
|
details: {
|
|
745
899
|
scope: "wiki_llm",
|
|
746
|
-
eventKey:
|
|
900
|
+
eventKey: useStoredBackgroundResponse
|
|
901
|
+
? "llm_compile_background_started"
|
|
902
|
+
: "llm_compile_foreground_completed",
|
|
747
903
|
provider: profile.provider,
|
|
748
904
|
baseUrl: profile.baseUrl,
|
|
749
905
|
model: profile.model,
|
|
@@ -755,7 +911,8 @@ export class OpenAiResponsesProvider {
|
|
|
755
911
|
}
|
|
756
912
|
let pollCount = 0;
|
|
757
913
|
let consecutivePollFailures = 0;
|
|
758
|
-
while (
|
|
914
|
+
while (useStoredBackgroundResponse &&
|
|
915
|
+
!isTerminalBackgroundStatus(readResponseStatus(payload))) {
|
|
759
916
|
await new Promise((resolve) => setTimeout(resolve, BACKGROUND_POLL_INTERVAL_MS));
|
|
760
917
|
try {
|
|
761
918
|
const pollResponse = await fetch(buildResponsesUrl(profile, responseId), {
|
|
@@ -838,7 +995,7 @@ export class OpenAiResponsesProvider {
|
|
|
838
995
|
}
|
|
839
996
|
}
|
|
840
997
|
const finalStatus = readResponseStatus(payload);
|
|
841
|
-
if (finalStatus !== "completed") {
|
|
998
|
+
if (finalStatus && finalStatus !== "completed") {
|
|
842
999
|
const errorMessage = readResponseError(payload) ??
|
|
843
1000
|
`OpenAI background wiki compilation ended with status ${finalStatus}.`;
|
|
844
1001
|
emitDiagnostic(logger, {
|
|
@@ -3,11 +3,10 @@ import { z } from "zod";
|
|
|
3
3
|
import { getDatabase } from "./db.js";
|
|
4
4
|
import { HttpError } from "./errors.js";
|
|
5
5
|
import { recordActivityEvent } from "./repositories/activity-events.js";
|
|
6
|
-
import {
|
|
6
|
+
import { getNoteById, updateNote } from "./repositories/notes.js";
|
|
7
7
|
import { createManualRewardGrant } from "./repositories/rewards.js";
|
|
8
8
|
import { listTaskRuns } from "./repositories/task-runs.js";
|
|
9
9
|
import { getDefaultUser } from "./repositories/users.js";
|
|
10
|
-
import { listWikiSpaces } from "./repositories/wiki-memory.js";
|
|
11
10
|
import { getScreenTimeOverlapSummary } from "./screen-time.js";
|
|
12
11
|
const movementPublishModeSchema = z.enum([
|
|
13
12
|
"auto_publish",
|
|
@@ -206,7 +205,7 @@ const movementTripInputSchema = z.object({
|
|
|
206
205
|
});
|
|
207
206
|
export const movementSettingsInputSchema = z.object({
|
|
208
207
|
trackingEnabled: z.boolean().default(false),
|
|
209
|
-
publishMode: movementPublishModeSchema.default("
|
|
208
|
+
publishMode: movementPublishModeSchema.default("draft_review"),
|
|
210
209
|
retentionMode: movementRetentionModeSchema.default("aggregates_only"),
|
|
211
210
|
locationPermissionStatus: z.string().trim().default("not_determined"),
|
|
212
211
|
motionPermissionStatus: z.string().trim().default("unknown"),
|
|
@@ -1030,7 +1029,7 @@ function defaultMovementSettings(userId) {
|
|
|
1030
1029
|
return {
|
|
1031
1030
|
userId,
|
|
1032
1031
|
trackingEnabled: false,
|
|
1033
|
-
publishMode: "
|
|
1032
|
+
publishMode: "draft_review",
|
|
1034
1033
|
retentionMode: "aggregates_only",
|
|
1035
1034
|
locationPermissionStatus: "not_determined",
|
|
1036
1035
|
motionPermissionStatus: "unknown",
|
|
@@ -1278,7 +1277,7 @@ function ensureMovementSettings(userId) {
|
|
|
1278
1277
|
background_tracking_ready, last_companion_sync_at, metadata_json,
|
|
1279
1278
|
created_at, updated_at
|
|
1280
1279
|
)
|
|
1281
|
-
VALUES (?, 0, '
|
|
1280
|
+
VALUES (?, 0, 'draft_review', 'aggregates_only', 'not_determined', 'unknown', 0, NULL, '{}', ?, ?)`)
|
|
1282
1281
|
.run(userId, now, now);
|
|
1283
1282
|
return getMovementSettingsRow(userId);
|
|
1284
1283
|
}
|
|
@@ -1363,9 +1362,6 @@ function listTripStops(tripIds) {
|
|
|
1363
1362
|
ORDER BY trip_id ASC, sequence_index ASC`)
|
|
1364
1363
|
.all(...tripIds);
|
|
1365
1364
|
}
|
|
1366
|
-
function defaultSpaceId() {
|
|
1367
|
-
return listWikiSpaces()[0]?.id;
|
|
1368
|
-
}
|
|
1369
1365
|
function syncPlaceWikiMetadata(placeId) {
|
|
1370
1366
|
const row = getDatabase()
|
|
1371
1367
|
.prepare(`SELECT *
|
|
@@ -1561,176 +1557,6 @@ function resolvePlaceForPatch(input) {
|
|
|
1561
1557
|
}
|
|
1562
1558
|
return undefined;
|
|
1563
1559
|
}
|
|
1564
|
-
function createMovementNote(input) {
|
|
1565
|
-
const spaceId = defaultSpaceId();
|
|
1566
|
-
if (!spaceId) {
|
|
1567
|
-
return null;
|
|
1568
|
-
}
|
|
1569
|
-
return createNote({
|
|
1570
|
-
kind: "evidence",
|
|
1571
|
-
title: input.title,
|
|
1572
|
-
slug: "",
|
|
1573
|
-
summary: "",
|
|
1574
|
-
contentMarkdown: input.contentMarkdown,
|
|
1575
|
-
spaceId,
|
|
1576
|
-
parentSlug: null,
|
|
1577
|
-
indexOrder: 0,
|
|
1578
|
-
showInIndex: false,
|
|
1579
|
-
aliases: [],
|
|
1580
|
-
userId: input.userId,
|
|
1581
|
-
author: null,
|
|
1582
|
-
links: [],
|
|
1583
|
-
tags: input.tags,
|
|
1584
|
-
destroyAt: null,
|
|
1585
|
-
sourcePath: "",
|
|
1586
|
-
frontmatter: input.frontmatter,
|
|
1587
|
-
revisionHash: ""
|
|
1588
|
-
}, { actor: "Movement sync", source: "system" });
|
|
1589
|
-
}
|
|
1590
|
-
function formatMovementDurationForNote(valueSeconds) {
|
|
1591
|
-
if (valueSeconds >= 86_400) {
|
|
1592
|
-
return `${round(valueSeconds / 86_400, 1)} days`;
|
|
1593
|
-
}
|
|
1594
|
-
if (valueSeconds >= 3_600) {
|
|
1595
|
-
return `${round(valueSeconds / 3_600, 1)} hours`;
|
|
1596
|
-
}
|
|
1597
|
-
return `${Math.max(1, Math.round(valueSeconds / 60))} minutes`;
|
|
1598
|
-
}
|
|
1599
|
-
function mergeMovementNoteTags(existingTags, existingFrontmatter, generatedTags) {
|
|
1600
|
-
const movement = existingFrontmatter.movement &&
|
|
1601
|
-
typeof existingFrontmatter.movement === "object" &&
|
|
1602
|
-
!Array.isArray(existingFrontmatter.movement)
|
|
1603
|
-
? existingFrontmatter.movement
|
|
1604
|
-
: null;
|
|
1605
|
-
const previousGeneratedTags = Array.isArray(movement?.generatedTags)
|
|
1606
|
-
? movement.generatedTags.filter((value) => typeof value === "string")
|
|
1607
|
-
: [];
|
|
1608
|
-
const previousGeneratedTagSet = new Set(previousGeneratedTags.map((tag) => tag.toLowerCase()));
|
|
1609
|
-
const preservedTags = existingTags.filter((tag) => !previousGeneratedTagSet.has(tag.toLowerCase()));
|
|
1610
|
-
return uniqStrings([...preservedTags, ...generatedTags]);
|
|
1611
|
-
}
|
|
1612
|
-
function syncMovementNote(input) {
|
|
1613
|
-
const existingNote = input.publishedNoteId
|
|
1614
|
-
? getNoteById(input.publishedNoteId)
|
|
1615
|
-
: null;
|
|
1616
|
-
if (existingNote && !Array.isArray(existingNote)) {
|
|
1617
|
-
const updated = updateNote(existingNote.id, {
|
|
1618
|
-
title: input.title,
|
|
1619
|
-
contentMarkdown: input.contentMarkdown,
|
|
1620
|
-
tags: mergeMovementNoteTags(existingNote.tags ?? [], existingNote.frontmatter, input.generatedTags),
|
|
1621
|
-
frontmatter: {
|
|
1622
|
-
...existingNote.frontmatter,
|
|
1623
|
-
...input.frontmatter
|
|
1624
|
-
}
|
|
1625
|
-
}, { actor: "Movement sync", source: "system" });
|
|
1626
|
-
return updated?.id ?? existingNote.id;
|
|
1627
|
-
}
|
|
1628
|
-
const created = createMovementNote({
|
|
1629
|
-
userId: input.userId,
|
|
1630
|
-
title: input.title,
|
|
1631
|
-
contentMarkdown: input.contentMarkdown,
|
|
1632
|
-
tags: input.generatedTags,
|
|
1633
|
-
frontmatter: input.frontmatter
|
|
1634
|
-
});
|
|
1635
|
-
return created?.id ?? null;
|
|
1636
|
-
}
|
|
1637
|
-
function syncStayNote(settings, stay, place) {
|
|
1638
|
-
if (!settings || settings.publishMode === "no_publish") {
|
|
1639
|
-
return null;
|
|
1640
|
-
}
|
|
1641
|
-
const label = place?.label || stay.label || "Unlabeled stay";
|
|
1642
|
-
const durationSecondsValue = durationSeconds(stay.started_at, stay.ended_at);
|
|
1643
|
-
const live = stay.status.trim().toLowerCase() !== "completed" &&
|
|
1644
|
-
stay.status.trim().toLowerCase() !== "closed";
|
|
1645
|
-
const generatedTags = uniqStrings([
|
|
1646
|
-
"movement",
|
|
1647
|
-
"stay",
|
|
1648
|
-
...(place ? safeJsonParse(place.category_tags_json, []) : [])
|
|
1649
|
-
]);
|
|
1650
|
-
const content = [
|
|
1651
|
-
live ? `Currently staying at **${label}**.` : `Stayed at **${label}**.`,
|
|
1652
|
-
"",
|
|
1653
|
-
`- Started: ${stay.started_at}`,
|
|
1654
|
-
`- ${live ? "Current end" : "Ended"}: ${stay.ended_at}`,
|
|
1655
|
-
`- Duration: ${formatMovementDurationForNote(durationSecondsValue)}`,
|
|
1656
|
-
`- Radius: ${Math.round(stay.radius_meters)} m`,
|
|
1657
|
-
`- Classification: ${stay.classification || "stationary"}`
|
|
1658
|
-
].join("\n");
|
|
1659
|
-
return syncMovementNote({
|
|
1660
|
-
userId: stay.user_id,
|
|
1661
|
-
publishedNoteId: stay.published_note_id,
|
|
1662
|
-
title: `Stay · ${label}`,
|
|
1663
|
-
contentMarkdown: content,
|
|
1664
|
-
generatedTags,
|
|
1665
|
-
frontmatter: {
|
|
1666
|
-
observedAt: stay.started_at,
|
|
1667
|
-
movement: {
|
|
1668
|
-
kind: "stay",
|
|
1669
|
-
state: live ? "live" : "closed",
|
|
1670
|
-
stayId: stay.id,
|
|
1671
|
-
publishMode: settings.publishMode,
|
|
1672
|
-
placeId: place?.id ?? null,
|
|
1673
|
-
placeLabel: label,
|
|
1674
|
-
startedAt: stay.started_at,
|
|
1675
|
-
endedAt: stay.ended_at,
|
|
1676
|
-
durationSeconds: durationSecondsValue,
|
|
1677
|
-
generatedTags
|
|
1678
|
-
}
|
|
1679
|
-
}
|
|
1680
|
-
});
|
|
1681
|
-
}
|
|
1682
|
-
function syncTripNote(settings, trip, startPlace, endPlace) {
|
|
1683
|
-
if (!settings || settings.publishMode === "no_publish") {
|
|
1684
|
-
return null;
|
|
1685
|
-
}
|
|
1686
|
-
const startLabel = startPlace?.label || "Unknown start";
|
|
1687
|
-
const endLabel = endPlace?.label || "Unknown end";
|
|
1688
|
-
const durationSecondsValue = durationSeconds(trip.started_at, trip.ended_at);
|
|
1689
|
-
const distanceKm = round(trip.distance_meters / 1000, 2);
|
|
1690
|
-
const live = trip.status.trim().toLowerCase() !== "completed" &&
|
|
1691
|
-
trip.status.trim().toLowerCase() !== "closed";
|
|
1692
|
-
const generatedTags = uniqStrings([
|
|
1693
|
-
"movement",
|
|
1694
|
-
"trip",
|
|
1695
|
-
...safeJsonParse(trip.tags_json, [])
|
|
1696
|
-
]);
|
|
1697
|
-
const content = [
|
|
1698
|
-
live
|
|
1699
|
-
? `Currently moving from **${startLabel}** to **${endLabel}**.`
|
|
1700
|
-
: `Travelled from **${startLabel}** to **${endLabel}**.`,
|
|
1701
|
-
"",
|
|
1702
|
-
`- Started: ${trip.started_at}`,
|
|
1703
|
-
`- ${live ? "Current end" : "Ended"}: ${trip.ended_at}`,
|
|
1704
|
-
`- Duration: ${formatMovementDurationForNote(durationSecondsValue)}`,
|
|
1705
|
-
`- Distance: ${distanceKm} km`,
|
|
1706
|
-
`- Activity: ${trip.activity_type || trip.travel_mode}`
|
|
1707
|
-
].join("\n");
|
|
1708
|
-
return syncMovementNote({
|
|
1709
|
-
userId: trip.user_id,
|
|
1710
|
-
publishedNoteId: trip.published_note_id,
|
|
1711
|
-
title: `Trip · ${startLabel} → ${endLabel}`,
|
|
1712
|
-
contentMarkdown: content,
|
|
1713
|
-
generatedTags,
|
|
1714
|
-
frontmatter: {
|
|
1715
|
-
observedAt: trip.started_at,
|
|
1716
|
-
movement: {
|
|
1717
|
-
kind: "trip",
|
|
1718
|
-
state: live ? "live" : "closed",
|
|
1719
|
-
tripId: trip.id,
|
|
1720
|
-
publishMode: settings.publishMode,
|
|
1721
|
-
startPlaceId: startPlace?.id ?? null,
|
|
1722
|
-
endPlaceId: endPlace?.id ?? null,
|
|
1723
|
-
startPlaceLabel: startLabel,
|
|
1724
|
-
endPlaceLabel: endLabel,
|
|
1725
|
-
startedAt: trip.started_at,
|
|
1726
|
-
endedAt: trip.ended_at,
|
|
1727
|
-
durationSeconds: durationSecondsValue,
|
|
1728
|
-
distanceMeters: trip.distance_meters,
|
|
1729
|
-
generatedTags
|
|
1730
|
-
}
|
|
1731
|
-
}
|
|
1732
|
-
});
|
|
1733
|
-
}
|
|
1734
1560
|
function awardMovementXp(input) {
|
|
1735
1561
|
const deltaXp = estimateMovementXp(input.categoryTags, input.distanceMeters);
|
|
1736
1562
|
if (deltaXp <= 0) {
|
|
@@ -1905,16 +1731,6 @@ function upsertMovementStay(pairing, settings, input) {
|
|
|
1905
1731
|
.prepare(`SELECT * FROM movement_stays WHERE user_id = ? AND external_uid = ?`)
|
|
1906
1732
|
.get(pairing.user_id, parsed.externalUid);
|
|
1907
1733
|
const freshMetadata = safeJsonParse(fresh.metadata_json, {});
|
|
1908
|
-
if (settings?.publishMode === "auto_publish" && !hasInvalidMovementRecord(freshMetadata)) {
|
|
1909
|
-
const publishedNoteId = syncStayNote(settings, fresh, matchedPlace);
|
|
1910
|
-
if (publishedNoteId && publishedNoteId !== fresh.published_note_id) {
|
|
1911
|
-
getDatabase()
|
|
1912
|
-
.prepare(`UPDATE movement_stays
|
|
1913
|
-
SET published_note_id = ?, updated_at = ?
|
|
1914
|
-
WHERE id = ?`)
|
|
1915
|
-
.run(publishedNoteId, nowIso(), fresh.id);
|
|
1916
|
-
}
|
|
1917
|
-
}
|
|
1918
1734
|
return {
|
|
1919
1735
|
mode: existing ? "updated" : "created",
|
|
1920
1736
|
stayId: fresh.id
|
|
@@ -2040,16 +1856,6 @@ function upsertMovementTrip(pairing, settings, input) {
|
|
|
2040
1856
|
.prepare(`SELECT * FROM movement_trips WHERE id = ?`)
|
|
2041
1857
|
.get(fresh.id);
|
|
2042
1858
|
const freshMetadata = safeJsonParse(refreshed.metadata_json, {});
|
|
2043
|
-
if (settings?.publishMode === "auto_publish" && !hasInvalidMovementRecord(freshMetadata)) {
|
|
2044
|
-
const publishedNoteId = syncTripNote(settings, refreshed, startPlace, endPlace);
|
|
2045
|
-
if (publishedNoteId && publishedNoteId !== refreshed.published_note_id) {
|
|
2046
|
-
getDatabase()
|
|
2047
|
-
.prepare(`UPDATE movement_trips
|
|
2048
|
-
SET published_note_id = ?, updated_at = ?
|
|
2049
|
-
WHERE id = ?`)
|
|
2050
|
-
.run(publishedNoteId, nowIso(), refreshed.id);
|
|
2051
|
-
}
|
|
2052
|
-
}
|
|
2053
1859
|
if (!existing && settings?.publishMode === "auto_publish" && !hasInvalidMovementRecord(freshMetadata)) {
|
|
2054
1860
|
awardMovementXp({
|
|
2055
1861
|
userId: pairing.user_id,
|
|
@@ -10,6 +10,8 @@ import { createNoteLinkSchema, crudEntityTypeSchema, noteKindSchema, noteSchema
|
|
|
10
10
|
import { deleteEncryptedSecret, readEncryptedSecret, storeEncryptedSecret } from "./calendar.js";
|
|
11
11
|
import { isEntityDeleted } from "./deleted-entities.js";
|
|
12
12
|
import { recordDiagnosticLog } from "./diagnostic-logs.js";
|
|
13
|
+
const MAX_WIKI_INGEST_TEXT_CHUNK_CHARS = 220_000;
|
|
14
|
+
const WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS = 2_000;
|
|
13
15
|
const wikiSpaceSchema = z.object({
|
|
14
16
|
id: z.string(),
|
|
15
17
|
slug: z.string(),
|
|
@@ -1515,6 +1517,7 @@ export function listWikiPages(query) {
|
|
|
1515
1517
|
ensureWikiSpaceSeedPages(spaceId);
|
|
1516
1518
|
return listAllNotes()
|
|
1517
1519
|
.filter((note) => note.spaceId === spaceId)
|
|
1520
|
+
.filter((note) => (query.includeHidden ? true : note.showInIndex))
|
|
1518
1521
|
.filter((note) => (query.kind ? note.kind === query.kind : true))
|
|
1519
1522
|
.sort(compareWikiPageOrder)
|
|
1520
1523
|
.slice(0, query.limit ?? 100);
|
|
@@ -1638,6 +1641,7 @@ export async function searchWikiPages(input, secrets) {
|
|
|
1638
1641
|
const parsed = wikiSearchQuerySchema.parse(input);
|
|
1639
1642
|
const pages = listAllNotes()
|
|
1640
1643
|
.filter((page) => (parsed.spaceId ? page.spaceId === parsed.spaceId : true))
|
|
1644
|
+
.filter((page) => page.showInIndex)
|
|
1641
1645
|
.filter((page) => (parsed.kind ? page.kind === parsed.kind : true));
|
|
1642
1646
|
const scores = new Map();
|
|
1643
1647
|
const addScore = (noteId, value) => {
|
|
@@ -2015,6 +2019,115 @@ export async function reindexWikiEmbeddings(input, secrets) {
|
|
|
2015
2019
|
chunkCount
|
|
2016
2020
|
};
|
|
2017
2021
|
}
|
|
2022
|
+
function isChunkableWikiIngestTextAsset(asset) {
|
|
2023
|
+
const mimeType = asset.mime_type.toLowerCase();
|
|
2024
|
+
const fileName = asset.file_name.toLowerCase();
|
|
2025
|
+
if (!existsSync(asset.file_path)) {
|
|
2026
|
+
return false;
|
|
2027
|
+
}
|
|
2028
|
+
if (asset.size_bytes <= MAX_WIKI_INGEST_TEXT_CHUNK_CHARS) {
|
|
2029
|
+
return false;
|
|
2030
|
+
}
|
|
2031
|
+
const metadata = parseJsonRecord(asset.metadata_json);
|
|
2032
|
+
if (metadata?.chunkParentAssetId || metadata?.textChunked) {
|
|
2033
|
+
return false;
|
|
2034
|
+
}
|
|
2035
|
+
return (mimeType.startsWith("text/") ||
|
|
2036
|
+
fileName.endsWith(".txt") ||
|
|
2037
|
+
fileName.endsWith(".md") ||
|
|
2038
|
+
fileName.endsWith(".markdown") ||
|
|
2039
|
+
fileName.endsWith(".csv") ||
|
|
2040
|
+
fileName.endsWith(".json"));
|
|
2041
|
+
}
|
|
2042
|
+
function splitWikiIngestTextIntoChunks(sourceText, maxChars) {
|
|
2043
|
+
const text = sourceText.trim();
|
|
2044
|
+
if (text.length <= maxChars) {
|
|
2045
|
+
return [text];
|
|
2046
|
+
}
|
|
2047
|
+
const chunks = [];
|
|
2048
|
+
let start = 0;
|
|
2049
|
+
while (start < text.length) {
|
|
2050
|
+
const hardEnd = Math.min(text.length, start + maxChars);
|
|
2051
|
+
let end = hardEnd;
|
|
2052
|
+
if (hardEnd < text.length) {
|
|
2053
|
+
const newline = text.lastIndexOf("\n", hardEnd);
|
|
2054
|
+
if (newline > start + Math.floor(maxChars * 0.65)) {
|
|
2055
|
+
end = newline + 1;
|
|
2056
|
+
}
|
|
2057
|
+
}
|
|
2058
|
+
const chunk = text.slice(start, end).trim();
|
|
2059
|
+
if (chunk.length > 0) {
|
|
2060
|
+
chunks.push(chunk);
|
|
2061
|
+
}
|
|
2062
|
+
if (end >= text.length) {
|
|
2063
|
+
break;
|
|
2064
|
+
}
|
|
2065
|
+
start = Math.max(0, end - WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS);
|
|
2066
|
+
}
|
|
2067
|
+
return chunks;
|
|
2068
|
+
}
|
|
2069
|
+
async function splitLargeWikiIngestTextAsset(options) {
|
|
2070
|
+
if (!isChunkableWikiIngestTextAsset(options.asset)) {
|
|
2071
|
+
return 0;
|
|
2072
|
+
}
|
|
2073
|
+
const sourceText = await readFile(options.asset.file_path, "utf8");
|
|
2074
|
+
const chunks = splitWikiIngestTextIntoChunks(sourceText, MAX_WIKI_INGEST_TEXT_CHUNK_CHARS);
|
|
2075
|
+
if (chunks.length <= 1) {
|
|
2076
|
+
return 0;
|
|
2077
|
+
}
|
|
2078
|
+
const extension = path.extname(options.asset.file_name) || ".txt";
|
|
2079
|
+
const baseName = path.basename(options.asset.file_name, extension) ||
|
|
2080
|
+
options.asset.file_name ||
|
|
2081
|
+
"source";
|
|
2082
|
+
const width = String(chunks.length).length;
|
|
2083
|
+
for (const [index, chunk] of chunks.entries()) {
|
|
2084
|
+
const chunkNumber = index + 1;
|
|
2085
|
+
const chunkFileName = `${baseName}-part-${String(chunkNumber).padStart(width, "0")}-of-${String(chunks.length).padStart(width, "0")}${extension}`;
|
|
2086
|
+
const chunkHeader = [
|
|
2087
|
+
`Source file: ${options.asset.file_name}`,
|
|
2088
|
+
`Source locator: ${options.asset.source_locator || options.asset.file_name}`,
|
|
2089
|
+
`Chunk: ${chunkNumber}/${chunks.length}`,
|
|
2090
|
+
`Parent checksum: ${options.asset.checksum}`,
|
|
2091
|
+
"",
|
|
2092
|
+
chunk
|
|
2093
|
+
].join("\n");
|
|
2094
|
+
const persisted = await persistIngestUpload({
|
|
2095
|
+
jobId: options.jobId,
|
|
2096
|
+
fileName: chunkFileName,
|
|
2097
|
+
mimeType: options.asset.mime_type || "text/plain",
|
|
2098
|
+
payload: Buffer.from(chunkHeader, "utf8")
|
|
2099
|
+
});
|
|
2100
|
+
createWikiIngestAssetRecord({
|
|
2101
|
+
jobId: options.jobId,
|
|
2102
|
+
sourceKind: "upload",
|
|
2103
|
+
sourceLocator: `${options.asset.source_locator || options.asset.file_name}#chunk-${chunkNumber}`,
|
|
2104
|
+
fileName: chunkFileName,
|
|
2105
|
+
mimeType: options.asset.mime_type || "text/plain",
|
|
2106
|
+
filePath: persisted.filePath,
|
|
2107
|
+
sizeBytes: persisted.sizeBytes,
|
|
2108
|
+
checksum: persisted.checksum,
|
|
2109
|
+
metadata: {
|
|
2110
|
+
chunkParentAssetId: options.asset.id,
|
|
2111
|
+
chunkParentChecksum: options.asset.checksum,
|
|
2112
|
+
chunkIndex: chunkNumber,
|
|
2113
|
+
chunkCount: chunks.length,
|
|
2114
|
+
chunkOverlapChars: WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS,
|
|
2115
|
+
chunkMaxChars: MAX_WIKI_INGEST_TEXT_CHUNK_CHARS
|
|
2116
|
+
}
|
|
2117
|
+
});
|
|
2118
|
+
}
|
|
2119
|
+
updateWikiIngestAsset(options.asset.id, {
|
|
2120
|
+
status: "completed",
|
|
2121
|
+
metadata: {
|
|
2122
|
+
...parseJsonRecord(options.asset.metadata_json),
|
|
2123
|
+
textChunked: true,
|
|
2124
|
+
textChunkCount: chunks.length,
|
|
2125
|
+
textChunkMaxChars: MAX_WIKI_INGEST_TEXT_CHUNK_CHARS,
|
|
2126
|
+
textChunkOverlapChars: WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS
|
|
2127
|
+
}
|
|
2128
|
+
});
|
|
2129
|
+
return chunks.length;
|
|
2130
|
+
}
|
|
2018
2131
|
function getWikiIngestJobDir(jobId) {
|
|
2019
2132
|
return path.join(resolveDataDir(), "wiki-ingest", jobId);
|
|
2020
2133
|
}
|
|
@@ -2613,7 +2726,10 @@ export async function processWikiIngestJob(jobId, options) {
|
|
|
2613
2726
|
const initialAssets = listWikiIngestJobAssetsInternal(jobId);
|
|
2614
2727
|
let processedFiles = initialAssets.filter((asset) => asset.status === "completed").length;
|
|
2615
2728
|
let totalFiles = Math.max(job.total_files, initialAssets.length);
|
|
2616
|
-
let hadSuccess =
|
|
2729
|
+
let hadSuccess = (() => {
|
|
2730
|
+
const counts = refreshCounts();
|
|
2731
|
+
return counts.pageCount + counts.entityCount > 0;
|
|
2732
|
+
})();
|
|
2617
2733
|
while (assetQueue().length > 0) {
|
|
2618
2734
|
const nextAsset = assetQueue().find((asset) => ["processing", "queued"].includes(asset.status));
|
|
2619
2735
|
if (!nextAsset) {
|
|
@@ -2675,6 +2791,27 @@ export async function processWikiIngestJob(jobId, options) {
|
|
|
2675
2791
|
}
|
|
2676
2792
|
continue;
|
|
2677
2793
|
}
|
|
2794
|
+
const derivedChunkCount = await splitLargeWikiIngestTextAsset({
|
|
2795
|
+
jobId,
|
|
2796
|
+
asset: nextAsset
|
|
2797
|
+
});
|
|
2798
|
+
if (derivedChunkCount > 0) {
|
|
2799
|
+
totalFiles = Math.max(derivedChunkCount, totalFiles - 1 + derivedChunkCount);
|
|
2800
|
+
updateWikiIngestJob(jobId, {
|
|
2801
|
+
totalFiles,
|
|
2802
|
+
latestMessage: `Split ${nextAsset.file_name || "large text source"} into ${derivedChunkCount} chunks.`
|
|
2803
|
+
});
|
|
2804
|
+
createWikiIngestLog(jobId, `Split ${nextAsset.file_name || "large text source"} into ${derivedChunkCount} chunks.`, "info", {
|
|
2805
|
+
sourceAssetId: nextAsset.id,
|
|
2806
|
+
fileName: nextAsset.file_name,
|
|
2807
|
+
sourceLocator: nextAsset.source_locator,
|
|
2808
|
+
checksum: nextAsset.checksum,
|
|
2809
|
+
chunkCount: derivedChunkCount,
|
|
2810
|
+
chunkMaxChars: MAX_WIKI_INGEST_TEXT_CHUNK_CHARS,
|
|
2811
|
+
chunkOverlapChars: WIKI_INGEST_TEXT_CHUNK_OVERLAP_CHARS
|
|
2812
|
+
});
|
|
2813
|
+
continue;
|
|
2814
|
+
}
|
|
2678
2815
|
updateWikiIngestAsset(nextAsset.id, { status: "processing" });
|
|
2679
2816
|
currentAssetContext = {
|
|
2680
2817
|
assetId: nextAsset.id,
|
|
@@ -30,10 +30,13 @@ export async function buildCompanionPairingTransport(input) {
|
|
|
30
30
|
pairPayload: snapshot.pairPayload,
|
|
31
31
|
alpn: snapshot.alpn ?? COMPANION_IROH_ALPN,
|
|
32
32
|
localBaseUrl: snapshot.localBaseUrl,
|
|
33
|
+
fallbackApiBaseUrl: requestApiBaseUrl,
|
|
34
|
+
fallbackUiBaseUrl: requestUiBaseUrl,
|
|
33
35
|
recreateCommand: snapshot.recreateCommand ?? undefined,
|
|
34
36
|
startedAt: snapshot.startedAt ?? undefined,
|
|
35
37
|
notes: [
|
|
36
|
-
"Default pairing uses Forge's Rust Iroh transport over QUIC.",
|
|
38
|
+
"Default pairing uses Forge's Rust Iroh transport over QUIC first.",
|
|
39
|
+
"The QR keeps the request API/UI URL as a direct fallback when Iroh cannot complete a request.",
|
|
37
40
|
"The QR payload carries the Iroh node id, host token, optional relay, and ALPN forge-companion/1.",
|
|
38
41
|
"Manual HTTP/TCP pairing remains available with --manual-http for advanced local setups."
|
|
39
42
|
]
|
|
@@ -200,12 +203,13 @@ function irohTransport(input) {
|
|
|
200
203
|
const nodeId = input.pairPayload.node_id;
|
|
201
204
|
return {
|
|
202
205
|
transportMode: "iroh",
|
|
203
|
-
apiBaseUrl:
|
|
204
|
-
uiBaseUrl:
|
|
206
|
+
apiBaseUrl: input.fallbackApiBaseUrl,
|
|
207
|
+
uiBaseUrl: input.fallbackUiBaseUrl,
|
|
205
208
|
transport: {
|
|
206
209
|
protocol: "iroh",
|
|
207
210
|
provider: "forge-companion-iroh",
|
|
208
211
|
status: "ready",
|
|
212
|
+
publicBaseUrl: input.fallbackApiBaseUrl,
|
|
209
213
|
localBaseUrl: input.localBaseUrl,
|
|
210
214
|
nodeId,
|
|
211
215
|
relay: input.pairPayload.relay,
|
|
@@ -297,6 +301,10 @@ function candidateIrohBinaries() {
|
|
|
297
301
|
return candidateIrohAssetRoots().flatMap((root) => [
|
|
298
302
|
path.join(root, "companion-iroh", "target", "release", binaryName),
|
|
299
303
|
path.join(root, "companion-iroh", "target", "debug", binaryName),
|
|
304
|
+
path.join(root, "companion-iroh-src", "target", "release", binaryName),
|
|
305
|
+
path.join(root, "companion-iroh-src", "target", "debug", binaryName),
|
|
306
|
+
path.join(root, "dist", "companion-iroh-src", "target", "release", binaryName),
|
|
307
|
+
path.join(root, "dist", "companion-iroh-src", "target", "debug", binaryName),
|
|
300
308
|
path.join(root, "openclaw-plugin", "dist", "companion-iroh", platformKey, binaryName),
|
|
301
309
|
path.join(root, "companion-iroh", platformKey, binaryName),
|
|
302
310
|
path.join(root, "companion-iroh", binaryName)
|
|
@@ -305,7 +313,8 @@ function candidateIrohBinaries() {
|
|
|
305
313
|
function resolveCompanionIrohManifestPath() {
|
|
306
314
|
const candidates = candidateIrohAssetRoots().flatMap((root) => [
|
|
307
315
|
path.join(root, "companion-iroh", "Cargo.toml"),
|
|
308
|
-
path.join(root, "companion-iroh-src", "Cargo.toml")
|
|
316
|
+
path.join(root, "companion-iroh-src", "Cargo.toml"),
|
|
317
|
+
path.join(root, "dist", "companion-iroh-src", "Cargo.toml")
|
|
309
318
|
]);
|
|
310
319
|
return candidates.find((candidate) => existsSync(candidate)) ?? null;
|
|
311
320
|
}
|
package/openclaw.plugin.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"id": "forge-openclaw-plugin",
|
|
3
3
|
"name": "Forge",
|
|
4
4
|
"description": "Curated OpenClaw adapter for the Forge collaboration API, UI entrypoint, and localhost auto-start runtime.",
|
|
5
|
-
"version": "0.2.
|
|
5
|
+
"version": "0.2.116",
|
|
6
6
|
"activation": {
|
|
7
7
|
"onStartup": true,
|
|
8
8
|
"onCapabilities": [
|
package/package.json
CHANGED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
UPDATE movement_stays
|
|
2
|
+
SET published_note_id = NULL
|
|
3
|
+
WHERE published_note_id IS NOT NULL;
|
|
4
|
+
|
|
5
|
+
UPDATE movement_trips
|
|
6
|
+
SET published_note_id = NULL
|
|
7
|
+
WHERE published_note_id IS NOT NULL;
|
|
8
|
+
|
|
9
|
+
DELETE FROM notes
|
|
10
|
+
WHERE source = 'system'
|
|
11
|
+
AND kind = 'evidence'
|
|
12
|
+
AND json_extract(frontmatter_json, '$.movement.kind') IN ('stay', 'trip');
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|