appback-remoteagent 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/runtime-env.js +10 -0
- package/dist/index.js +6 -2
- package/dist/services/bridge-service.js +83 -27
- package/dist/services/codex-usage-fallback-service.js +111 -0
- package/dist/services/provider-setup-service.js +2 -4
- package/dist/store/file-store.js +3 -0
- package/docs/CLI_BOOTSTRAP_AND_SECRET_MIGRATION.md +2 -0
- package/docs/ERROR_NORMALIZATION.md +14 -0
- package/docs/OPERATIONS.md +2 -2
- package/docs/RELEASING.md +22 -3
- package/docs/SERVER26_UBUNTU_MIGRATION_HANDOVER.md +350 -0
- package/package.json +2 -1
- package/scripts/release-deploy.sh +39 -4
- package/scripts/release-publish.sh +20 -5
- package/scripts/selftest-cli.mjs +18 -0
- package/scripts/selftest-model-fallback.mjs +132 -0
- package/scripts/selftest-telegram-update.mjs +45 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import os from "node:os";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
import process from "node:process";
|
|
3
4
|
import { fileURLToPath } from "node:url";
|
|
4
5
|
const CHILD_ENV_BLOCKED_PREFIXES = ["TELEGRAM_"];
|
|
5
6
|
export function buildProviderEnv(extraEnv) {
|
|
@@ -15,11 +16,20 @@ export function buildProviderEnv(extraEnv) {
|
|
|
15
16
|
env.DATA_DIR = dataDir;
|
|
16
17
|
env.REMOTEAGENT_DATA_DIR = dataDir;
|
|
17
18
|
env.REMOTEAGENT_SECRET_BIN = resolveSecretHelperPath();
|
|
19
|
+
env.PATH = buildRuntimePath(env.PATH);
|
|
18
20
|
if (extraEnv) {
|
|
19
21
|
Object.assign(env, extraEnv);
|
|
20
22
|
}
|
|
21
23
|
return env;
|
|
22
24
|
}
|
|
25
|
+
export function buildRuntimePath(existingPath = process.env.PATH, nodeExecutable = process.execPath, homeDir = os.homedir()) {
|
|
26
|
+
const entries = [
|
|
27
|
+
path.dirname(nodeExecutable),
|
|
28
|
+
path.join(homeDir, ".local", "bin"),
|
|
29
|
+
...(existingPath ?? "").split(path.delimiter),
|
|
30
|
+
].filter(Boolean);
|
|
31
|
+
return [...new Set(entries)].join(path.delimiter);
|
|
32
|
+
}
|
|
23
33
|
function resolveSecretHelperPath() {
|
|
24
34
|
const adapterDir = path.dirname(fileURLToPath(import.meta.url));
|
|
25
35
|
return path.resolve(adapterDir, "..", "secret-helper.js");
|
package/dist/index.js
CHANGED
|
@@ -19,6 +19,7 @@ import { BotPollingStateService } from "./services/bot-polling-state-service.js"
|
|
|
19
19
|
import { ProviderRecoveryService } from "./services/provider-recovery-service.js";
|
|
20
20
|
import { computePolicyPollIntervalMs, computeRecentMessageRanks } from "./services/polling-policy.js";
|
|
21
21
|
import { terminateAllSpawnedExecutions } from "./adapters/windows-shell.js";
|
|
22
|
+
import { buildProviderEnv } from "./adapters/runtime-env.js";
|
|
22
23
|
import { setTelegramCommandMenu } from "./telegram-command-menu.js";
|
|
23
24
|
import { buildBotInfoFromIdentity, buildFallbackBotInfo } from "./telegram-bot-identity.js";
|
|
24
25
|
const execFileAsync = promisify(execFile);
|
|
@@ -528,9 +529,12 @@ function commandExists(command) {
|
|
|
528
529
|
return fs.existsSync(trimmed);
|
|
529
530
|
}
|
|
530
531
|
if (process.platform === "win32") {
|
|
531
|
-
return spawnSync("where", [trimmed], { stdio: "ignore" }).status === 0;
|
|
532
|
+
return spawnSync("where", [trimmed], { stdio: "ignore", env: buildProviderEnv() }).status === 0;
|
|
532
533
|
}
|
|
533
|
-
return spawnSync("sh", ["-lc", 'command -v "$0" >/dev/null 2>&1', trimmed], {
|
|
534
|
+
return spawnSync("sh", ["-lc", 'command -v "$0" >/dev/null 2>&1', trimmed], {
|
|
535
|
+
stdio: "ignore",
|
|
536
|
+
env: buildProviderEnv(),
|
|
537
|
+
}).status === 0;
|
|
534
538
|
}
|
|
535
539
|
async function acquireProcessLock(dataDir) {
|
|
536
540
|
await fsp.mkdir(dataDir, { recursive: true });
|
|
@@ -3,8 +3,9 @@ import { randomBytes } from "node:crypto";
|
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { stopSpawnedExecution } from "../adapters/windows-shell.js";
|
|
6
|
+
import { CODEX_USAGE_FALLBACK_MODEL, CodexUsageFallbackService, parseCodexUsageLimit, } from "./codex-usage-fallback-service.js";
|
|
6
7
|
const MODEL_PRESETS = {
|
|
7
|
-
codex: ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.6", "gpt-5.5", "gpt-5.4", "gpt-5.4-mini", "gpt-5.2", "gpt-5.1-codex-max"],
|
|
8
|
+
codex: ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.6", "gpt-5.5", "gpt-5.4", "gpt-5.4-mini", "gpt-5.3-codex-spark", "gpt-5.2", "gpt-5.1-codex-max"],
|
|
8
9
|
claude: ["sonnet", "opus", "haiku"],
|
|
9
10
|
};
|
|
10
11
|
const DEFAULT_CODEX_MODEL = "gpt-5.6-sol";
|
|
@@ -17,6 +18,7 @@ export class BridgeService {
|
|
|
17
18
|
preferredStartMode;
|
|
18
19
|
defaultCodexSandboxMode;
|
|
19
20
|
sessionLocks = new Map();
|
|
21
|
+
codexUsageFallback;
|
|
20
22
|
constructor(store, adapters, defaultWorkspace, workspaceRoot, isProviderInstalled, preferredStartMode, defaultCodexSandboxMode) {
|
|
21
23
|
this.store = store;
|
|
22
24
|
this.adapters = adapters;
|
|
@@ -25,6 +27,7 @@ export class BridgeService {
|
|
|
25
27
|
this.isProviderInstalled = isProviderInstalled;
|
|
26
28
|
this.preferredStartMode = preferredStartMode;
|
|
27
29
|
this.defaultCodexSandboxMode = defaultCodexSandboxMode;
|
|
30
|
+
this.codexUsageFallback = new CodexUsageFallbackService(store.getDataDir());
|
|
28
31
|
}
|
|
29
32
|
async createSession(botId, chatId, cwd) {
|
|
30
33
|
const provider = await this.resolveStartProvider();
|
|
@@ -490,7 +493,33 @@ export class BridgeService {
|
|
|
490
493
|
for (const provider of providers) {
|
|
491
494
|
const providerSession = this.ensureProviderSession(session, provider);
|
|
492
495
|
this.ensureConfigured(provider);
|
|
493
|
-
const
|
|
496
|
+
const primaryModel = providerSession.model ?? this.defaultModelFor(provider);
|
|
497
|
+
const selection = provider === "codex"
|
|
498
|
+
? await this.codexUsageFallback.selectExecutionModel(primaryModel)
|
|
499
|
+
: { model: primaryModel, recoveryProbe: false };
|
|
500
|
+
let executionModel = selection.model;
|
|
501
|
+
const forwardProgress = async (output, model) => {
|
|
502
|
+
const progressResponse = {
|
|
503
|
+
provider,
|
|
504
|
+
sessionId: providerSession.sessionId ?? session.sessionId,
|
|
505
|
+
publicSessionId: session.publicId,
|
|
506
|
+
model,
|
|
507
|
+
cwd: providerSession.cwd,
|
|
508
|
+
output,
|
|
509
|
+
};
|
|
510
|
+
await this.log({
|
|
511
|
+
timestamp: new Date().toISOString(),
|
|
512
|
+
remoteSessionId: session.sessionId,
|
|
513
|
+
botId,
|
|
514
|
+
chatId,
|
|
515
|
+
provider,
|
|
516
|
+
direction: "out",
|
|
517
|
+
sessionId: providerSession.sessionId,
|
|
518
|
+
text: output,
|
|
519
|
+
});
|
|
520
|
+
await onProgress?.(progressResponse);
|
|
521
|
+
};
|
|
522
|
+
const sendWithModel = (model) => this.adapters[provider].send({
|
|
494
523
|
botId,
|
|
495
524
|
chatId: chatId ?? requestSource,
|
|
496
525
|
remoteSessionId: session.sessionId,
|
|
@@ -498,32 +527,59 @@ export class BridgeService {
|
|
|
498
527
|
cwd: providerSession.cwd,
|
|
499
528
|
sessionId: providerSession.sessionId,
|
|
500
529
|
message,
|
|
501
|
-
model
|
|
530
|
+
model,
|
|
502
531
|
sandboxMode: providerSession.sandboxMode,
|
|
503
|
-
onProgress: onProgress
|
|
504
|
-
? async (output) => {
|
|
505
|
-
const progressResponse = {
|
|
506
|
-
provider,
|
|
507
|
-
sessionId: providerSession.sessionId ?? session.sessionId,
|
|
508
|
-
publicSessionId: session.publicId,
|
|
509
|
-
model: providerSession.model ?? this.defaultModelFor(provider),
|
|
510
|
-
cwd: providerSession.cwd,
|
|
511
|
-
output,
|
|
512
|
-
};
|
|
513
|
-
await this.log({
|
|
514
|
-
timestamp: new Date().toISOString(),
|
|
515
|
-
remoteSessionId: session.sessionId,
|
|
516
|
-
botId,
|
|
517
|
-
chatId,
|
|
518
|
-
provider,
|
|
519
|
-
direction: "out",
|
|
520
|
-
sessionId: providerSession.sessionId,
|
|
521
|
-
text: output,
|
|
522
|
-
});
|
|
523
|
-
await onProgress(progressResponse);
|
|
524
|
-
}
|
|
525
|
-
: undefined,
|
|
532
|
+
onProgress: onProgress ? (output) => forwardProgress(output, model) : undefined,
|
|
526
533
|
});
|
|
534
|
+
const reportHarnessNotice = async (output, model) => {
|
|
535
|
+
try {
|
|
536
|
+
await forwardProgress(output, model);
|
|
537
|
+
}
|
|
538
|
+
catch (error) {
|
|
539
|
+
if (error instanceof Error && error.name === "SilentTelegramAbort") {
|
|
540
|
+
throw error;
|
|
541
|
+
}
|
|
542
|
+
const messageText = error instanceof Error ? error.message : String(error);
|
|
543
|
+
console.warn(`[codex-usage-fallback] notice delivery failed for ${session.publicId}: ${messageText}`);
|
|
544
|
+
await this.log({
|
|
545
|
+
timestamp: new Date().toISOString(),
|
|
546
|
+
remoteSessionId: session.sessionId,
|
|
547
|
+
botId,
|
|
548
|
+
chatId,
|
|
549
|
+
provider: "system",
|
|
550
|
+
direction: "system",
|
|
551
|
+
sessionId: providerSession.sessionId,
|
|
552
|
+
text: `Codex usage fallback notice delivery failed: ${messageText}`,
|
|
553
|
+
});
|
|
554
|
+
}
|
|
555
|
+
};
|
|
556
|
+
let response;
|
|
557
|
+
try {
|
|
558
|
+
response = await sendWithModel(executionModel);
|
|
559
|
+
}
|
|
560
|
+
catch (error) {
|
|
561
|
+
const messageText = error instanceof Error ? error.message : String(error);
|
|
562
|
+
const usageLimit = provider === "codex" ? parseCodexUsageLimit(messageText) : undefined;
|
|
563
|
+
if (!usageLimit || executionModel === CODEX_USAGE_FALLBACK_MODEL) {
|
|
564
|
+
throw error;
|
|
565
|
+
}
|
|
566
|
+
await this.codexUsageFallback.activate(usageLimit);
|
|
567
|
+
executionModel = CODEX_USAGE_FALLBACK_MODEL;
|
|
568
|
+
const resetText = usageLimit.resetAtLabel ? ` 초기화 예정: ${usageLimit.resetAtLabel}.` : "";
|
|
569
|
+
await reportHarnessNotice([
|
|
570
|
+
"REPORT:progress",
|
|
571
|
+
`기본 모델 ${primaryModel}의 사용 한도가 소진되어 이번 실행을 ${executionModel}로 임시 전환합니다.${resetText}`,
|
|
572
|
+
"세션의 기본 모델 설정은 변경하지 않습니다.",
|
|
573
|
+
].join("\n"), executionModel);
|
|
574
|
+
response = await sendWithModel(executionModel);
|
|
575
|
+
}
|
|
576
|
+
if (provider === "codex" && selection.recoveryProbe && executionModel === primaryModel) {
|
|
577
|
+
await this.codexUsageFallback.clear();
|
|
578
|
+
await reportHarnessNotice([
|
|
579
|
+
"REPORT:progress",
|
|
580
|
+
`사용 한도 초기화 후 기본 모델 ${primaryModel} 호출이 성공하여 원래 모델로 복귀했습니다.`,
|
|
581
|
+
].join("\n"), primaryModel);
|
|
582
|
+
}
|
|
527
583
|
const updatedProviderSession = {
|
|
528
584
|
...providerSession,
|
|
529
585
|
cwd: response.cwd,
|
|
@@ -553,7 +609,7 @@ export class BridgeService {
|
|
|
553
609
|
responses.push({
|
|
554
610
|
...response,
|
|
555
611
|
publicSessionId: session.publicId,
|
|
556
|
-
model:
|
|
612
|
+
model: executionModel,
|
|
557
613
|
});
|
|
558
614
|
}
|
|
559
615
|
return responses;
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
export const CODEX_USAGE_FALLBACK_MODEL = "gpt-5.3-codex-spark";
|
|
5
|
+
export class CodexUsageFallbackService {
|
|
6
|
+
now;
|
|
7
|
+
statePath;
|
|
8
|
+
writeTail = Promise.resolve();
|
|
9
|
+
constructor(dataDir, now = () => new Date()) {
|
|
10
|
+
this.now = now;
|
|
11
|
+
this.statePath = path.join(dataDir, "codex-usage-fallback.json");
|
|
12
|
+
}
|
|
13
|
+
async selectExecutionModel(primaryModel) {
|
|
14
|
+
if (primaryModel === CODEX_USAGE_FALLBACK_MODEL) {
|
|
15
|
+
return { model: primaryModel, recoveryProbe: false };
|
|
16
|
+
}
|
|
17
|
+
const state = await this.read();
|
|
18
|
+
if (!state) {
|
|
19
|
+
return { model: primaryModel, recoveryProbe: false };
|
|
20
|
+
}
|
|
21
|
+
if (Date.parse(state.resetAt) > this.now().getTime()) {
|
|
22
|
+
return { model: state.fallbackModel, recoveryProbe: false };
|
|
23
|
+
}
|
|
24
|
+
return { model: primaryModel, recoveryProbe: true };
|
|
25
|
+
}
|
|
26
|
+
async activate(limit) {
|
|
27
|
+
if (!limit.resetAt || Date.parse(limit.resetAt) <= this.now().getTime()) {
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
await this.write({
|
|
31
|
+
fallbackModel: CODEX_USAGE_FALLBACK_MODEL,
|
|
32
|
+
activatedAt: this.now().toISOString(),
|
|
33
|
+
resetAt: limit.resetAt,
|
|
34
|
+
resetAtLabel: limit.resetAtLabel,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
async clear() {
|
|
38
|
+
await this.withWriteLock(async () => {
|
|
39
|
+
await fs.unlink(this.statePath).catch((error) => {
|
|
40
|
+
if (error.code !== "ENOENT") {
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
async read() {
|
|
47
|
+
const raw = await fs.readFile(this.statePath, "utf8").catch((error) => {
|
|
48
|
+
if (error.code === "ENOENT") {
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
throw error;
|
|
52
|
+
});
|
|
53
|
+
if (!raw) {
|
|
54
|
+
return undefined;
|
|
55
|
+
}
|
|
56
|
+
try {
|
|
57
|
+
const parsed = JSON.parse(raw);
|
|
58
|
+
if (parsed.fallbackModel !== CODEX_USAGE_FALLBACK_MODEL
|
|
59
|
+
|| typeof parsed.activatedAt !== "string"
|
|
60
|
+
|| typeof parsed.resetAt !== "string"
|
|
61
|
+
|| !Number.isFinite(Date.parse(parsed.resetAt))) {
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
return parsed;
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
async write(state) {
|
|
71
|
+
await this.withWriteLock(async () => {
|
|
72
|
+
await fs.mkdir(path.dirname(this.statePath), { recursive: true });
|
|
73
|
+
const temporaryPath = `${this.statePath}.${process.pid}.${randomUUID()}.tmp`;
|
|
74
|
+
await fs.writeFile(temporaryPath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 });
|
|
75
|
+
await fs.rename(temporaryPath, this.statePath);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
async withWriteLock(task) {
|
|
79
|
+
const previous = this.writeTail;
|
|
80
|
+
let release;
|
|
81
|
+
this.writeTail = new Promise((resolve) => {
|
|
82
|
+
release = resolve;
|
|
83
|
+
});
|
|
84
|
+
await previous.catch(() => undefined);
|
|
85
|
+
try {
|
|
86
|
+
await task();
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
release();
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
export function parseCodexUsageLimit(message) {
|
|
94
|
+
if (!/you(?:'|’)ve hit your usage limit\b/i.test(message)) {
|
|
95
|
+
return undefined;
|
|
96
|
+
}
|
|
97
|
+
const resetMatch = /try again at\s+(.+?)(?:\.\s|\.?$|\n)/i.exec(message);
|
|
98
|
+
const resetAtLabel = resetMatch?.[1]?.trim();
|
|
99
|
+
if (!resetAtLabel) {
|
|
100
|
+
return {};
|
|
101
|
+
}
|
|
102
|
+
const normalized = resetAtLabel.replace(/(\d{1,2})(?:st|nd|rd|th)\b/gi, "$1");
|
|
103
|
+
const timestamp = Date.parse(normalized);
|
|
104
|
+
if (!Number.isFinite(timestamp)) {
|
|
105
|
+
return { resetAtLabel };
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
resetAt: new Date(timestamp).toISOString(),
|
|
109
|
+
resetAtLabel,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
@@ -3,6 +3,7 @@ import os from "node:os";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import process from "node:process";
|
|
5
5
|
import { spawn } from "node:child_process";
|
|
6
|
+
import { buildProviderEnv } from "../adapters/runtime-env.js";
|
|
6
7
|
export class ProviderSetupService {
|
|
7
8
|
timeoutMs;
|
|
8
9
|
isProviderAvailable;
|
|
@@ -224,10 +225,7 @@ export class ProviderSetupService {
|
|
|
224
225
|
return new Promise((resolve, reject) => {
|
|
225
226
|
const child = spawn(launcher.file, launcher.args, {
|
|
226
227
|
cwd: process.cwd(),
|
|
227
|
-
env:
|
|
228
|
-
...process.env,
|
|
229
|
-
...extraEnv,
|
|
230
|
-
},
|
|
228
|
+
env: buildProviderEnv(extraEnv),
|
|
231
229
|
});
|
|
232
230
|
let stdout = "";
|
|
233
231
|
let stderr = "";
|
package/dist/store/file-store.js
CHANGED
|
@@ -51,6 +51,9 @@ export class FileStore {
|
|
|
51
51
|
.filter((session) => sessionIds.has(session.sessionId))
|
|
52
52
|
.sort((left, right) => right.updatedAt.localeCompare(left.updatedAt));
|
|
53
53
|
}
|
|
54
|
+
getDataDir() {
|
|
55
|
+
return this.dataDir;
|
|
56
|
+
}
|
|
54
57
|
async listActiveSessionIds(botId) {
|
|
55
58
|
const state = await this.readState();
|
|
56
59
|
const sessionIds = Object.values(state.chats)
|
|
@@ -32,6 +32,8 @@ remoteagent-start
|
|
|
32
32
|
sudo systemctl restart remoteagent
|
|
33
33
|
```
|
|
34
34
|
|
|
35
|
+
Provider install, login, and execution commands automatically prepend the directory of the Node executable running RemoteAgent and `~/.local/bin` to their child-process `PATH`. This keeps `/install codex`, `/install claude`, and `/login` working when RemoteAgent runs under systemd with Node installed through nvm, even if the service itself was started with a minimal system PATH.
|
|
36
|
+
|
|
35
37
|
## Secret migration
|
|
36
38
|
|
|
37
39
|
RemoteAgent `/secret` values belong to the installation, not to an individual agent session. They are stored under `~/.remoteagent/managed/secrets.json`.
|
|
@@ -21,6 +21,12 @@ RemoteAgent should never leak raw provider event payloads directly to Telegram u
|
|
|
21
21
|
- `provider.timeout.final`
|
|
22
22
|
- example: `Codex timed out after 600s without returning a final reply`
|
|
23
23
|
- behavior: do not retry automatically; explain the exact configured timeout that killed the provider process
|
|
24
|
+
- `provider.usage_limit.fallback`
|
|
25
|
+
- example: `You've hit your usage limit ... try again at Aug 27th, 2026 3:52 AM`
|
|
26
|
+
- behavior: preserve the session's primary model and retry the same request once with `gpt-5.3-codex-spark`
|
|
27
|
+
- scope: the persisted fallback window belongs to the machine's Codex account, not to an individual RemoteAgent session
|
|
28
|
+
- recovery: after the reported reset time, try the primary model on the next request and clear fallback state only after that call succeeds
|
|
29
|
+
- exclusions: model capacity, HTTP 429, Telegram rate limits, 503 responses, and transport errors must not activate this fallback
|
|
24
30
|
|
|
25
31
|
## Current terminal behavior
|
|
26
32
|
|
|
@@ -36,6 +42,14 @@ When a provider process reaches `COMMAND_TIMEOUT_MS`:
|
|
|
36
42
|
- do not invent a cause such as context compaction, provider outage, or model quality unless the provider output explicitly says that
|
|
37
43
|
- tell the operator to increase `/option timeout <seconds>` for long-running work
|
|
38
44
|
|
|
45
|
+
When Codex reports an explicit account usage-limit error:
|
|
46
|
+
|
|
47
|
+
- store the temporary state in `codex-usage-fallback.json`
|
|
48
|
+
- do not modify the model stored in any session
|
|
49
|
+
- use the same Codex thread id, workspace, prompt, and sandbox while overriding only the execution model
|
|
50
|
+
- do not retry recursively if the fallback model also reports a usage-limit error
|
|
51
|
+
- if the error does not contain a parseable future reset time, use Spark only for the current execution and probe the primary model again on the next request
|
|
52
|
+
|
|
39
53
|
## Maintenance rule
|
|
40
54
|
|
|
41
55
|
When a new provider error shape appears:
|
package/docs/OPERATIONS.md
CHANGED
|
@@ -191,8 +191,8 @@ ssh au2223@192.168.0.30 'export PATH="/home/au2223/.local/bin:/home/au2223/.nvm/
|
|
|
191
191
|
For server 26:
|
|
192
192
|
|
|
193
193
|
```bash
|
|
194
|
-
ssh ospadmin@192.168.
|
|
195
|
-
ssh ospadmin@192.168.
|
|
194
|
+
ssh ospadmin@192.168.33.26 'export PATH="$HOME/.local/bin:$PATH"; npm explore -g appback-remoteagent -- npm run maintenance:disk -- report'
|
|
195
|
+
ssh ospadmin@192.168.33.26 'export PATH="$HOME/.local/bin:$PATH"; npm explore -g appback-remoteagent -- npm run maintenance:disk -- prune-safe'
|
|
196
196
|
```
|
|
197
197
|
|
|
198
198
|
## Git workflow
|
package/docs/RELEASING.md
CHANGED
|
@@ -66,7 +66,8 @@ npm run release:publish
|
|
|
66
66
|
|
|
67
67
|
The publish script performs:
|
|
68
68
|
|
|
69
|
-
-
|
|
69
|
+
- tracked working tree check
|
|
70
|
+
- a clean staging snapshot from committed `HEAD`; unrelated untracked files are listed but excluded
|
|
70
71
|
- `npm whoami`
|
|
71
72
|
- `npm owner ls appback-remoteagent`
|
|
72
73
|
- `npm run check`
|
|
@@ -86,6 +87,12 @@ Deploy to server 30:
|
|
|
86
87
|
npm run release:deploy -- <version> 30
|
|
87
88
|
```
|
|
88
89
|
|
|
90
|
+
Deploy to server 40:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
npm run release:deploy -- <version> 40
|
|
94
|
+
```
|
|
95
|
+
|
|
89
96
|
Deploy to server 26:
|
|
90
97
|
|
|
91
98
|
```bash
|
|
@@ -108,6 +115,7 @@ The deploy script performs:
|
|
|
108
115
|
|
|
109
116
|
- npm registry version check for `appback-remoteagent@<version>`
|
|
110
117
|
- server 30 npm install, install hook, systemd restart, version/log verification
|
|
118
|
+
- server 40 npm install, install hook, user-process restart, version/log verification
|
|
111
119
|
- server 26 npm install, install hook, user-process restart, version/log verification
|
|
112
120
|
|
|
113
121
|
## 6. Verify
|
|
@@ -121,7 +129,7 @@ npm view appback-remoteagent version
|
|
|
121
129
|
Server 30:
|
|
122
130
|
|
|
123
131
|
```bash
|
|
124
|
-
ssh au2223@192.168.
|
|
132
|
+
ssh au2223@192.168.33.30 'bash -lc '"'"'
|
|
125
133
|
export PATH="/home/au2223/.local/bin:/home/au2223/.nvm/versions/node/v22.22.0/bin:$PATH"
|
|
126
134
|
systemctl is-active remoteagent
|
|
127
135
|
node -p "require(\"/home/au2223/.nvm/versions/node/v22.22.0/lib/node_modules/appback-remoteagent/package.json\").version"
|
|
@@ -132,7 +140,18 @@ journalctl -u remoteagent --since "5 minutes ago" --no-pager | tail -80
|
|
|
132
140
|
Server 26:
|
|
133
141
|
|
|
134
142
|
```bash
|
|
135
|
-
ssh ospadmin@192.168.
|
|
143
|
+
ssh ospadmin@192.168.33.26 'bash -lc '"'"'
|
|
144
|
+
npm list -g appback-remoteagent --depth=0
|
|
145
|
+
pgrep -af "appback-remoteagent/dist/index.js"
|
|
146
|
+
tail -80 ~/.remoteagent/logs/agent.log
|
|
147
|
+
'"'"''
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Server 40:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
ssh appback@192.168.33.40 'bash -lc '"'"'
|
|
154
|
+
export PATH="$HOME/.local/bin:$HOME/.nvm/versions/node/v22.23.2/bin:$PATH"
|
|
136
155
|
npm list -g appback-remoteagent --depth=0
|
|
137
156
|
pgrep -af "appback-remoteagent/dist/index.js"
|
|
138
157
|
tail -80 ~/.remoteagent/logs/agent.log
|
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
# Server 26 Ubuntu Migration Handover
|
|
2
|
+
|
|
3
|
+
## 1. Purpose
|
|
4
|
+
|
|
5
|
+
This document hands over the conversion of server 26 from the legacy
|
|
6
|
+
RHEL/SQream/backup host into the primary production application server.
|
|
7
|
+
|
|
8
|
+
Target role:
|
|
9
|
+
|
|
10
|
+
- public address: `125.132.17.159`
|
|
11
|
+
- internal address: `192.168.33.26`
|
|
12
|
+
- SSH account before reinstall: `ospadmin`
|
|
13
|
+
- proposed hostname after reinstall: `appback-prod`
|
|
14
|
+
- proposed OS: a clean Ubuntu LTS installation
|
|
15
|
+
- primary workload: production application containers
|
|
16
|
+
- external disk: backup-only storage
|
|
17
|
+
|
|
18
|
+
The internal address should remain `192.168.33.26`. There is no operational
|
|
19
|
+
benefit in changing it again, and retaining it reduces changes to NAT,
|
|
20
|
+
firewall, SSH, monitoring, and deployment configuration.
|
|
21
|
+
|
|
22
|
+
This document does not authorize destructive work. Formatting or reinstalling
|
|
23
|
+
the OS disk requires explicit user approval after the backup and restore gates
|
|
24
|
+
below pass.
|
|
25
|
+
|
|
26
|
+
## 2. Current Verified State
|
|
27
|
+
|
|
28
|
+
Verified on `2026-08-13` through:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
ssh ospadmin@192.168.33.26
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
### Host and network
|
|
35
|
+
|
|
36
|
+
- current OS: Red Hat Enterprise Linux 8.9
|
|
37
|
+
- current hostname: `scailium-node`
|
|
38
|
+
- active interface: `enp0s31f6`
|
|
39
|
+
- active address: `192.168.33.26/24`
|
|
40
|
+
- current wired MAC: `1c:1b:0d:90:8b:ce`
|
|
41
|
+
- former Wi-Fi profile: `DAONEWF5G`
|
|
42
|
+
- former Wi-Fi interface: `wlp0s20f0u5`
|
|
43
|
+
- former cloned Wi-Fi MAC: `00:15:5D:BC:FA:34`
|
|
44
|
+
- DNS and outbound Telegram API access are currently working
|
|
45
|
+
|
|
46
|
+
Confirm that the router maps public address `125.132.17.159` to this host
|
|
47
|
+
before production cutover. Do not assume that the public address is configured
|
|
48
|
+
directly on the Linux interface.
|
|
49
|
+
|
|
50
|
+
### Disks
|
|
51
|
+
|
|
52
|
+
| Device | Size | Filesystem | Current role |
|
|
53
|
+
| --- | ---: | --- | --- |
|
|
54
|
+
| `/dev/sdc` | 238.5GB | RHEL LVM/XFS | Current OS disk; intended Ubuntu install target |
|
|
55
|
+
| `/dev/sda2` | 3.7TB | XFS, label `RA_BACKUP_4T` | Mounted at `/mnt/backup`; preserve |
|
|
56
|
+
| `/dev/sdb1` | 1.8TB | NTFS | Currently unmounted; ownership and purpose must be confirmed |
|
|
57
|
+
|
|
58
|
+
Current filesystem pressure:
|
|
59
|
+
|
|
60
|
+
- `/`: 207GB total, 185GB used, 90%
|
|
61
|
+
- `/home`: 23GB total, 14GB used, 63%
|
|
62
|
+
- `/mnt/backup`: 3.7TB total, 73GB used, 2%
|
|
63
|
+
|
|
64
|
+
Physically disconnect both `/dev/sda` and `/dev/sdb` before installing Ubuntu.
|
|
65
|
+
Only `/dev/sdc` may be selected by the installer.
|
|
66
|
+
|
|
67
|
+
## 3. Critical Backup Finding
|
|
68
|
+
|
|
69
|
+
`/mnt/backup/pg` exists but is currently empty. The PostgreSQL dumps that must
|
|
70
|
+
be preserved are still on the OS-side storage under `/data/cluster`.
|
|
71
|
+
|
|
72
|
+
Verified dump inventory:
|
|
73
|
+
|
|
74
|
+
| Source | Size | Dump count | Latest verified dump |
|
|
75
|
+
| --- | ---: | ---: | --- |
|
|
76
|
+
| `/data/cluster/appback-hub-backups` | 8.7GB | 141 | `2026-08-09 06:17 KST`, about 90MB |
|
|
77
|
+
| `/data/cluster/grid-clash-backups` | 30GB | 122 | `2026-08-09 07:49 KST`, about 409MB |
|
|
78
|
+
| `/data/cluster/damoa-backups` | 16GB | 4 | `2026-08-09 02:48 KST`, about 4.3GB |
|
|
79
|
+
|
|
80
|
+
These dumps predate the current boot and are not yet confirmed as complete or
|
|
81
|
+
restorable. File existence is not a restore test.
|
|
82
|
+
|
|
83
|
+
The external disk currently contains media backup data instead:
|
|
84
|
+
|
|
85
|
+
- `/mnt/backup/remoteagent/damoa-media/current`: about 2.3GB
|
|
86
|
+
- `/mnt/backup/remoteagent/damoa-media/history`: about 44GB
|
|
87
|
+
- `/mnt/backup/remoteagent/damoa-media/restore-tests`: empty
|
|
88
|
+
|
|
89
|
+
### Mandatory copy stage
|
|
90
|
+
|
|
91
|
+
Copy the database backup trees to the external disk without deleting either
|
|
92
|
+
source or destination data:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
sudo mkdir -p /mnt/backup/pg
|
|
96
|
+
sudo rsync -aH --info=progress2 \
|
|
97
|
+
/data/cluster/appback-hub-backups/ \
|
|
98
|
+
/mnt/backup/pg/appback-hub-backups/
|
|
99
|
+
sudo rsync -aH --info=progress2 \
|
|
100
|
+
/data/cluster/grid-clash-backups/ \
|
|
101
|
+
/mnt/backup/pg/grid-clash-backups/
|
|
102
|
+
sudo rsync -aH --info=progress2 \
|
|
103
|
+
/data/cluster/damoa-backups/ \
|
|
104
|
+
/mnt/backup/pg/damoa-backups/
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Generate a manifest after the copy:
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
cd /mnt/backup/pg
|
|
111
|
+
find . -type f -name '*.dump' -print0 \
|
|
112
|
+
| sort -z \
|
|
113
|
+
| xargs -0 sha256sum \
|
|
114
|
+
> SHA256SUMS
|
|
115
|
+
sha256sum -c SHA256SUMS
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
The copy stage is complete only when file counts, total sizes, and SHA-256
|
|
119
|
+
verification match the source inventory.
|
|
120
|
+
|
|
121
|
+
### Mandatory restore stage
|
|
122
|
+
|
|
123
|
+
For each service, restore the newest selected dump into an isolated PostgreSQL
|
|
124
|
+
instance and record:
|
|
125
|
+
|
|
126
|
+
- PostgreSQL version used for restore
|
|
127
|
+
- `pg_restore` exit status
|
|
128
|
+
- restored database size
|
|
129
|
+
- schema count and expected key tables
|
|
130
|
+
- basic row-count checks agreed with the service owner
|
|
131
|
+
- application health check against the restored database where possible
|
|
132
|
+
|
|
133
|
+
Store the evidence under:
|
|
134
|
+
|
|
135
|
+
```text
|
|
136
|
+
/mnt/backup/pg/restore-evidence/<service>/<timestamp>/
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Do not reinstall the OS until all required databases have a successful restore
|
|
140
|
+
record. If the latest dump fails, stop and report it rather than silently using
|
|
141
|
+
an older dump.
|
|
142
|
+
|
|
143
|
+
## 4. Current RemoteAgent State
|
|
144
|
+
|
|
145
|
+
RemoteAgent was repaired during the network investigation:
|
|
146
|
+
|
|
147
|
+
- package: `appback-remoteagent@0.21.1`
|
|
148
|
+
- runtime: user `systemd` service
|
|
149
|
+
- unit: `~/.config/systemd/user/remoteagent.service`
|
|
150
|
+
- state: active and enabled
|
|
151
|
+
- linger: enabled for `ospadmin`
|
|
152
|
+
- providers detected: Codex
|
|
153
|
+
- configured Telegram bots observed:
|
|
154
|
+
- `@bot26pcbotbotbot_bot`
|
|
155
|
+
- `@ayeTemail_bot`
|
|
156
|
+
|
|
157
|
+
Runtime data is under:
|
|
158
|
+
|
|
159
|
+
```text
|
|
160
|
+
/home/ospadmin/.remoteagent
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
The Secret Store contains operational credentials. Secret values must never be
|
|
164
|
+
written into this repository or the handover report. Export it through the
|
|
165
|
+
RemoteAgent secret migration procedure and store the encrypted archive on the
|
|
166
|
+
external disk with mode `600`.
|
|
167
|
+
|
|
168
|
+
Also preserve:
|
|
169
|
+
|
|
170
|
+
- `~/.ssh/authorized_keys`
|
|
171
|
+
- the current RemoteAgent `.env`
|
|
172
|
+
- session/state data only if existing Telegram sessions must be resumed
|
|
173
|
+
- `/home/ospadmin/crontab.before-network-fix-20260813-094632`
|
|
174
|
+
|
|
175
|
+
Reinstall RemoteAgent from npm after Ubuntu is ready. Do not copy the old npm
|
|
176
|
+
global package directory from RHEL.
|
|
177
|
+
|
|
178
|
+
## 5. SQream and Scheduled Jobs
|
|
179
|
+
|
|
180
|
+
SQream is not healthy and must not be treated as a running database.
|
|
181
|
+
|
|
182
|
+
Actions already completed:
|
|
183
|
+
|
|
184
|
+
- active SQream configuration IPs were changed from `192.168.0.26` to
|
|
185
|
+
`192.168.33.26`
|
|
186
|
+
- overlapping market/strategy processes were terminated
|
|
187
|
+
- the affected market cron lines were commented with `TEMP_NETWORK_FIX`
|
|
188
|
+
- the original crontab was backed up at the path recorded above
|
|
189
|
+
|
|
190
|
+
Current SQream state:
|
|
191
|
+
|
|
192
|
+
- metadata server is reachable on port `3105`
|
|
193
|
+
- server picker is reachable on port `3108`
|
|
194
|
+
- active SQream workers: `0`
|
|
195
|
+
- all five workers connect to metadata and then exit with:
|
|
196
|
+
|
|
197
|
+
```text
|
|
198
|
+
Error: mismatch in mac address validation Please contact SQream support.
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
The license appears tied to the former Wi-Fi MAC. Reissuing the license for the
|
|
202
|
+
wired MAC would be required if SQream were retained. The target server role no
|
|
203
|
+
longer requires SQream, so the recommended migration is to preserve only data
|
|
204
|
+
that has an explicit owner and then omit SQream from the new Ubuntu build.
|
|
205
|
+
|
|
206
|
+
Do not resume the paused market cron jobs on the current host. Decide separately
|
|
207
|
+
whether those jobs are retired or moved to a dedicated analytics host.
|
|
208
|
+
|
|
209
|
+
## 6. Recommended Migration Plan
|
|
210
|
+
|
|
211
|
+
### Phase A: inventory and freeze
|
|
212
|
+
|
|
213
|
+
1. Confirm the application services that will run on the new production host.
|
|
214
|
+
2. Confirm whether all three PostgreSQL backup families must be retained.
|
|
215
|
+
3. Confirm the owner and disposition of `/dev/sdb1`.
|
|
216
|
+
4. Record router NAT, port forwarding, firewall, DNS, and TLS settings.
|
|
217
|
+
5. Freeze configuration changes during the final backup window.
|
|
218
|
+
|
|
219
|
+
Completion evidence:
|
|
220
|
+
|
|
221
|
+
- signed-off service list
|
|
222
|
+
- signed-off data retention list
|
|
223
|
+
- exported network configuration
|
|
224
|
+
- approved maintenance window
|
|
225
|
+
|
|
226
|
+
### Phase B: preserve and test
|
|
227
|
+
|
|
228
|
+
1. Copy all retained dumps to `/mnt/backup/pg`.
|
|
229
|
+
2. Generate and verify checksums.
|
|
230
|
+
3. Perform actual isolated restores.
|
|
231
|
+
4. Back up SSH access, encrypted RemoteAgent secrets, and required configuration.
|
|
232
|
+
5. Copy the migration evidence to a second machine or storage target.
|
|
233
|
+
|
|
234
|
+
Completion evidence:
|
|
235
|
+
|
|
236
|
+
- successful checksum log
|
|
237
|
+
- successful restore evidence for every retained database
|
|
238
|
+
- two independent copies of critical credentials and restore evidence
|
|
239
|
+
|
|
240
|
+
### Phase C: clean Ubuntu installation
|
|
241
|
+
|
|
242
|
+
1. Shut down the host.
|
|
243
|
+
2. Physically disconnect the 3.7TB external disk and the 1.8TB NTFS disk.
|
|
244
|
+
3. Install a mature Ubuntu LTS release on `/dev/sdc` only.
|
|
245
|
+
4. Set hostname `appback-prod`.
|
|
246
|
+
5. Configure static address `192.168.33.26/24` and gateway
|
|
247
|
+
`192.168.33.1`.
|
|
248
|
+
6. Apply security updates and configure time synchronization.
|
|
249
|
+
7. Restore SSH access and verify a second administrative session before
|
|
250
|
+
closing the installation console.
|
|
251
|
+
|
|
252
|
+
The installer record must include the selected target disk and partition map.
|
|
253
|
+
|
|
254
|
+
### Phase D: production runtime
|
|
255
|
+
|
|
256
|
+
Recommended filesystem ownership:
|
|
257
|
+
|
|
258
|
+
```text
|
|
259
|
+
/opt/appback/compose deployment definitions
|
|
260
|
+
/srv/appback/data persistent application data
|
|
261
|
+
/srv/appback/logs application logs
|
|
262
|
+
/mnt/backup external backup disk
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
Install only the required runtime components:
|
|
266
|
+
|
|
267
|
+
- Docker Engine and Compose plugin
|
|
268
|
+
- reverse proxy such as Nginx or Caddy
|
|
269
|
+
- production application stack
|
|
270
|
+
- monitoring and log rotation
|
|
271
|
+
- RemoteAgent from the published npm package, if it remains required on 26
|
|
272
|
+
|
|
273
|
+
Expose only HTTP/HTTPS publicly. Keep SSH and database ports restricted to the
|
|
274
|
+
internal network or VPN. Verify whether `125.132.17.159` is routed by NAT before
|
|
275
|
+
changing host firewall rules.
|
|
276
|
+
|
|
277
|
+
### Phase E: restore and cutover
|
|
278
|
+
|
|
279
|
+
1. Reconnect the external disk after Ubuntu is fully installed.
|
|
280
|
+
2. Mount it by UUID at `/mnt/backup`; do not reformat it.
|
|
281
|
+
3. Restore application databases from the verified backup set.
|
|
282
|
+
4. Deploy applications from versioned images/configuration.
|
|
283
|
+
5. Run service, database, TLS, upload, and external access checks.
|
|
284
|
+
6. Enable scheduled backups only after a manual backup and restore test passes.
|
|
285
|
+
7. Change DNS/NAT only during the approved cutover window.
|
|
286
|
+
|
|
287
|
+
## 7. Acceptance Criteria
|
|
288
|
+
|
|
289
|
+
The migration is complete only when all of the following are recorded:
|
|
290
|
+
|
|
291
|
+
- Ubuntu boots with the intended hostname and static internal address
|
|
292
|
+
- SSH works from an authorized internal host
|
|
293
|
+
- only approved public ports are exposed
|
|
294
|
+
- each production container has a versioned image and restart policy
|
|
295
|
+
- each required database was restored from a verified dump
|
|
296
|
+
- application health and representative user flows pass
|
|
297
|
+
- backup writes to the external disk
|
|
298
|
+
- a post-install restore test succeeds
|
|
299
|
+
- disk, CPU, memory, container, and certificate monitoring is active
|
|
300
|
+
- RemoteAgent, if installed, runs as exactly one supervised service
|
|
301
|
+
- no SQream or legacy market cron process remains unless separately approved
|
|
302
|
+
|
|
303
|
+
## 8. Stop Conditions
|
|
304
|
+
|
|
305
|
+
Stop and report before any destructive step if any of these is true:
|
|
306
|
+
|
|
307
|
+
- `/mnt/backup/pg` has not been populated and checksum-verified
|
|
308
|
+
- a required database has no successful restore test
|
|
309
|
+
- the external and NTFS disks cannot be positively distinguished from the OS disk
|
|
310
|
+
- public-IP NAT ownership is unclear
|
|
311
|
+
- required credentials or SSH recovery access have only one copy
|
|
312
|
+
- the application inventory or retention decision is incomplete
|
|
313
|
+
- an unexpected active workload is found on server 26
|
|
314
|
+
|
|
315
|
+
## 9. Rollback
|
|
316
|
+
|
|
317
|
+
Before OS installation, rollback means leaving RHEL unchanged and keeping the
|
|
318
|
+
paused cron jobs paused.
|
|
319
|
+
|
|
320
|
+
After OS installation, rollback cannot restore the old RHEL installation unless
|
|
321
|
+
a full disk image was created. The operational rollback is therefore:
|
|
322
|
+
|
|
323
|
+
1. keep the prior production server serving traffic
|
|
324
|
+
2. restore the verified database backup to the prior or replacement host
|
|
325
|
+
3. revert DNS/NAT to the prior production endpoint
|
|
326
|
+
4. collect Ubuntu deployment logs before attempting another cutover
|
|
327
|
+
|
|
328
|
+
Do not designate server 26 as the sole production host until this rollback path
|
|
329
|
+
has been tested.
|
|
330
|
+
|
|
331
|
+
## 10. First Commands for the Next Agent
|
|
332
|
+
|
|
333
|
+
Use these read-only checks before changing anything:
|
|
334
|
+
|
|
335
|
+
```bash
|
|
336
|
+
ssh ospadmin@192.168.33.26
|
|
337
|
+
hostnamectl
|
|
338
|
+
ip -br addr
|
|
339
|
+
lsblk -o NAME,SIZE,FSTYPE,LABEL,UUID,MOUNTPOINT
|
|
340
|
+
df -hT / /home /mnt/backup
|
|
341
|
+
systemctl --user status remoteagent --no-pager
|
|
342
|
+
pgrep -af 'sqream|remoteagent|run_kr_strategy|run_toss_kr'
|
|
343
|
+
crontab -l
|
|
344
|
+
find /data/cluster -type f -name '*.dump' -printf '%TY-%Tm-%Td %TH:%TM %s %p\n' | sort | tail
|
|
345
|
+
find /mnt/backup/pg -type f -name '*.dump' -printf '%TY-%Tm-%Td %TH:%TM %s %p\n' | sort | tail
|
|
346
|
+
```
|
|
347
|
+
|
|
348
|
+
The next agent should first report the current state and the exact phase it is
|
|
349
|
+
starting. No formatting, deletion, package removal, or cron reactivation should
|
|
350
|
+
occur merely from reading this document.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "appback-remoteagent",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.22.0",
|
|
4
4
|
"description": "Personal installable session server for continuing local AI work across PC and Telegram",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
"check": "tsc --noEmit -p tsconfig.json",
|
|
21
21
|
"selftest:telegram": "npm run build && node scripts/selftest-telegram-update.mjs",
|
|
22
22
|
"selftest:codex-stream": "npm run build && node scripts/selftest-codex-stream.mjs",
|
|
23
|
+
"selftest:model-fallback": "npm run build && node scripts/selftest-model-fallback.mjs",
|
|
23
24
|
"selftest:cli": "npm run build && node scripts/selftest-cli.mjs",
|
|
24
25
|
"prepare": "npm run build",
|
|
25
26
|
"prepublishOnly": "node scripts/prepublish-guard.mjs",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
set -euo pipefail
|
|
3
3
|
|
|
4
4
|
usage() {
|
|
5
|
-
echo "Usage: npm run release:deploy -- <version> <30|26|all>" >&2
|
|
5
|
+
echo "Usage: npm run release:deploy -- <version> <30|40|26|all>" >&2
|
|
6
6
|
echo "Example: npm run release:deploy -- 0.15.5 all" >&2
|
|
7
7
|
}
|
|
8
8
|
|
|
@@ -20,7 +20,7 @@ if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
|
|
20
20
|
fi
|
|
21
21
|
|
|
22
22
|
case "$TARGET" in
|
|
23
|
-
30|26|all)
|
|
23
|
+
30|40|26|all)
|
|
24
24
|
;;
|
|
25
25
|
*)
|
|
26
26
|
usage
|
|
@@ -36,7 +36,7 @@ if [[ "$REGISTRY_VERSION" != "$VERSION" ]]; then
|
|
|
36
36
|
fi
|
|
37
37
|
|
|
38
38
|
deploy_30() {
|
|
39
|
-
ssh au2223@192.168.
|
|
39
|
+
ssh au2223@192.168.33.30 "VERSION=$VERSION bash -s" <<'REMOTE'
|
|
40
40
|
set -euo pipefail
|
|
41
41
|
export PATH="/home/au2223/.local/bin:/home/au2223/.nvm/versions/node/v22.22.0/bin:$PATH"
|
|
42
42
|
node - <<'NODE'
|
|
@@ -63,7 +63,7 @@ REMOTE
|
|
|
63
63
|
}
|
|
64
64
|
|
|
65
65
|
deploy_26() {
|
|
66
|
-
ssh ospadmin@192.168.
|
|
66
|
+
ssh ospadmin@192.168.33.26 "VERSION=$VERSION bash -s" <<'REMOTE'
|
|
67
67
|
set -euo pipefail
|
|
68
68
|
export PATH="$HOME/.local/bin:$PATH"
|
|
69
69
|
node - <<'NODE'
|
|
@@ -93,15 +93,50 @@ tail -80 ~/.remoteagent/logs/agent.log
|
|
|
93
93
|
REMOTE
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
deploy_40() {
|
|
97
|
+
ssh appback@192.168.33.40 "VERSION=$VERSION bash -s" <<'REMOTE'
|
|
98
|
+
set -euo pipefail
|
|
99
|
+
export PATH="$HOME/.local/bin:$HOME/.nvm/versions/node/v22.23.2/bin:$PATH"
|
|
100
|
+
node - <<'NODE'
|
|
101
|
+
const fs = require("fs");
|
|
102
|
+
const path = `${process.env.HOME}/.remoteagent/bot-polling-state.json`;
|
|
103
|
+
if (fs.existsSync(path)) {
|
|
104
|
+
const state = JSON.parse(fs.readFileSync(path, "utf8"));
|
|
105
|
+
const running = Object.values(state.bots || {})
|
|
106
|
+
.filter((bot) => Array.isArray(bot.runningSessionIds) && bot.runningSessionIds.length > 0)
|
|
107
|
+
.map((bot) => `${bot.username || bot.botId}: ${bot.runningSessionIds.join(", ")}`);
|
|
108
|
+
if (running.length > 0) {
|
|
109
|
+
console.error("RemoteAgent has active provider work. Retry deploy after it finishes:");
|
|
110
|
+
for (const item of running) console.error(`- ${item}`);
|
|
111
|
+
process.exit(2);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
NODE
|
|
115
|
+
npm install -g "appback-remoteagent@$VERSION"
|
|
116
|
+
remoteagent-install
|
|
117
|
+
~/.remoteagent/stop-remoteagent.sh || true
|
|
118
|
+
sleep 2
|
|
119
|
+
~/.remoteagent/start-remoteagent.sh
|
|
120
|
+
sleep 5
|
|
121
|
+
npm list -g appback-remoteagent --depth=0
|
|
122
|
+
pgrep -af 'appback-remoteagent/dist/index.js'
|
|
123
|
+
tail -80 ~/.remoteagent/logs/agent.log
|
|
124
|
+
REMOTE
|
|
125
|
+
}
|
|
126
|
+
|
|
96
127
|
case "$TARGET" in
|
|
97
128
|
30)
|
|
98
129
|
deploy_30
|
|
99
130
|
;;
|
|
131
|
+
40)
|
|
132
|
+
deploy_40
|
|
133
|
+
;;
|
|
100
134
|
26)
|
|
101
135
|
deploy_26
|
|
102
136
|
;;
|
|
103
137
|
all)
|
|
104
138
|
deploy_30
|
|
139
|
+
deploy_40
|
|
105
140
|
deploy_26
|
|
106
141
|
;;
|
|
107
142
|
esac
|
|
@@ -12,12 +12,19 @@ if [[ "$PACKAGE_NAME" != "appback-remoteagent" ]]; then
|
|
|
12
12
|
exit 1
|
|
13
13
|
fi
|
|
14
14
|
|
|
15
|
-
if [[ -n "$(git status --porcelain)" ]]; then
|
|
16
|
-
echo "
|
|
17
|
-
git status --short >&2
|
|
15
|
+
if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then
|
|
16
|
+
echo "Tracked files are not clean. Commit and push before publishing." >&2
|
|
17
|
+
git status --short --untracked-files=no >&2
|
|
18
18
|
exit 1
|
|
19
19
|
fi
|
|
20
20
|
|
|
21
|
+
UNTRACKED="$(git ls-files --others --exclude-standard)"
|
|
22
|
+
if [[ -n "$UNTRACKED" ]]; then
|
|
23
|
+
echo "Untracked files are excluded from the release snapshot:"
|
|
24
|
+
printf '%s\n' "$UNTRACKED"
|
|
25
|
+
echo
|
|
26
|
+
fi
|
|
27
|
+
|
|
21
28
|
echo "Publishing $PACKAGE_NAME@$VERSION"
|
|
22
29
|
echo
|
|
23
30
|
echo "npm identity:"
|
|
@@ -30,12 +37,20 @@ echo
|
|
|
30
37
|
npm run check
|
|
31
38
|
npm run build
|
|
32
39
|
|
|
33
|
-
|
|
40
|
+
STAGING_DIR="$(mktemp -d)"
|
|
34
41
|
cleanup() {
|
|
35
|
-
rm -
|
|
42
|
+
rm -rf "$STAGING_DIR"
|
|
36
43
|
}
|
|
37
44
|
trap cleanup EXIT
|
|
38
45
|
|
|
46
|
+
git archive --format=tar HEAD | tar -xf - -C "$STAGING_DIR"
|
|
47
|
+
cd "$STAGING_DIR"
|
|
48
|
+
npm ci --ignore-scripts
|
|
49
|
+
npm run check
|
|
50
|
+
npm run build
|
|
51
|
+
|
|
52
|
+
TARBALL="$(npm pack --silent)"
|
|
53
|
+
|
|
39
54
|
echo
|
|
40
55
|
echo "Publishing tarball: $TARBALL"
|
|
41
56
|
REMOTEAGENT_PUBLISH_GUARD_OK=1 npm publish "$TARBALL" --access public
|
package/scripts/selftest-cli.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
registerTelegramBot,
|
|
10
10
|
waitForTelegramOwner,
|
|
11
11
|
} from "../dist/services/cli-config-service.js";
|
|
12
|
+
import { buildProviderEnv, buildRuntimePath } from "../dist/adapters/runtime-env.js";
|
|
12
13
|
import { exportSecrets, importSecrets } from "../dist/services/secret-transfer-service.js";
|
|
13
14
|
|
|
14
15
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "remoteagent-cli-selftest-"));
|
|
@@ -19,6 +20,23 @@ const selectedBundlePath = path.join(root, "selected-transfer.ra-secrets");
|
|
|
19
20
|
const passphrase = "correct-horse-battery-staple";
|
|
20
21
|
|
|
21
22
|
try {
|
|
23
|
+
const expectedNodeBin = path.join(root, "nvm", "bin");
|
|
24
|
+
const expectedHome = path.join(root, "home");
|
|
25
|
+
const runtimePath = buildRuntimePath(
|
|
26
|
+
["/usr/local/bin", "/usr/bin", "/usr/bin"].join(path.delimiter),
|
|
27
|
+
path.join(expectedNodeBin, "node"),
|
|
28
|
+
expectedHome,
|
|
29
|
+
).split(path.delimiter);
|
|
30
|
+
assert.deepEqual(runtimePath, [
|
|
31
|
+
expectedNodeBin,
|
|
32
|
+
path.join(expectedHome, ".local", "bin"),
|
|
33
|
+
"/usr/local/bin",
|
|
34
|
+
"/usr/bin",
|
|
35
|
+
]);
|
|
36
|
+
const providerEnv = buildProviderEnv();
|
|
37
|
+
assert.equal(providerEnv.PATH?.split(path.delimiter)[0], path.dirname(process.execPath));
|
|
38
|
+
assert.ok(providerEnv.PATH?.split(path.delimiter).includes(path.join(os.homedir(), ".local", "bin")));
|
|
39
|
+
|
|
22
40
|
const binDir = path.join(root, "bin");
|
|
23
41
|
const curlArgsPath = path.join(root, "curl-args.txt");
|
|
24
42
|
const curlUpdateCallsPath = path.join(root, "curl-update-calls.txt");
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import assert from "node:assert/strict";
|
|
4
|
+
import fs from "node:fs/promises";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
|
|
8
|
+
import { BridgeService } from "../dist/services/bridge-service.js";
|
|
9
|
+
import {
|
|
10
|
+
CODEX_USAGE_FALLBACK_MODEL,
|
|
11
|
+
CodexUsageFallbackService,
|
|
12
|
+
parseCodexUsageLimit,
|
|
13
|
+
} from "../dist/services/codex-usage-fallback-service.js";
|
|
14
|
+
import { FileStore } from "../dist/store/file-store.js";
|
|
15
|
+
|
|
16
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "remoteagent-model-fallback-"));
|
|
17
|
+
const dataDir = path.join(root, "data");
|
|
18
|
+
const workspaceRoot = path.join(root, "workspaces");
|
|
19
|
+
const defaultWorkspace = path.join(root, "default-workspace");
|
|
20
|
+
const fallbackStatePath = path.join(dataDir, "codex-usage-fallback.json");
|
|
21
|
+
const primaryModel = "gpt-5.6-sol";
|
|
22
|
+
const usageError = "You've hit your usage limit. Visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Aug 27th, 2099 3:52 AM.";
|
|
23
|
+
|
|
24
|
+
try {
|
|
25
|
+
await fs.mkdir(defaultWorkspace, { recursive: true });
|
|
26
|
+
const parsed = parseCodexUsageLimit(usageError);
|
|
27
|
+
assert.ok(parsed?.resetAt);
|
|
28
|
+
assert.equal(parsed.resetAtLabel, "Aug 27th, 2099 3:52 AM");
|
|
29
|
+
assert.equal(parseCodexUsageLimit("Selected model is at capacity."), undefined);
|
|
30
|
+
assert.equal(parseCodexUsageLimit("Too Many Requests: retry after 5"), undefined);
|
|
31
|
+
|
|
32
|
+
const store = new FileStore(dataDir, "codex");
|
|
33
|
+
await store.init();
|
|
34
|
+
const calls = [];
|
|
35
|
+
const progress = [];
|
|
36
|
+
let failPrimaryWithUsageLimit = true;
|
|
37
|
+
let failFallbackWithUsageLimit = false;
|
|
38
|
+
const provider = {
|
|
39
|
+
async send(request) {
|
|
40
|
+
calls.push({ model: request.model, message: request.message, sessionId: request.sessionId });
|
|
41
|
+
if (request.model === primaryModel && failPrimaryWithUsageLimit) {
|
|
42
|
+
throw new Error(usageError);
|
|
43
|
+
}
|
|
44
|
+
if (request.model === CODEX_USAGE_FALLBACK_MODEL && failFallbackWithUsageLimit) {
|
|
45
|
+
throw new Error("You've hit your usage limit. Try again at Aug 28th, 2099 1:00 AM.");
|
|
46
|
+
}
|
|
47
|
+
return {
|
|
48
|
+
provider: "codex",
|
|
49
|
+
sessionId: request.sessionId || "thread-model-fallback",
|
|
50
|
+
cwd: request.cwd,
|
|
51
|
+
output: "REPORT:result\nmodel fallback self-test completed",
|
|
52
|
+
};
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
const createBridge = () => new BridgeService(
|
|
56
|
+
store,
|
|
57
|
+
{ codex: provider },
|
|
58
|
+
defaultWorkspace,
|
|
59
|
+
workspaceRoot,
|
|
60
|
+
(providerName) => providerName === "codex",
|
|
61
|
+
"codex",
|
|
62
|
+
"workspace-write",
|
|
63
|
+
);
|
|
64
|
+
|
|
65
|
+
let bridge = createBridge();
|
|
66
|
+
const started = await bridge.startSession("test-bot", "test-chat", "codex");
|
|
67
|
+
const originalSessionId = started.session.sessionId;
|
|
68
|
+
const first = await bridge.routeMessage("test-bot", "test-chat", "first request", async (response) => {
|
|
69
|
+
progress.push(response.output);
|
|
70
|
+
});
|
|
71
|
+
assert.deepEqual(calls.map((call) => call.model), [primaryModel, CODEX_USAGE_FALLBACK_MODEL]);
|
|
72
|
+
assert.deepEqual(calls.map((call) => call.message), ["first request", "first request"]);
|
|
73
|
+
assert.equal(first[0]?.model, CODEX_USAGE_FALLBACK_MODEL);
|
|
74
|
+
assert.match(progress.join("\n"), /임시 전환합니다/);
|
|
75
|
+
|
|
76
|
+
const afterFallback = await store.getChatSession("test-bot", "test-chat");
|
|
77
|
+
assert.equal(afterFallback?.session.sessionId, originalSessionId);
|
|
78
|
+
assert.equal(afterFallback?.session.codex?.model, primaryModel);
|
|
79
|
+
assert.ok(await fs.stat(fallbackStatePath));
|
|
80
|
+
|
|
81
|
+
calls.length = 0;
|
|
82
|
+
bridge = createBridge();
|
|
83
|
+
const otherSession = await bridge.startSession("other-bot", "other-chat", "codex");
|
|
84
|
+
await bridge.routeMessage("other-bot", "other-chat", "second request");
|
|
85
|
+
assert.deepEqual(calls.map((call) => call.model), [CODEX_USAGE_FALLBACK_MODEL]);
|
|
86
|
+
assert.equal(calls[0]?.sessionId, undefined);
|
|
87
|
+
const otherStoredSession = await store.getChatSession("other-bot", "other-chat");
|
|
88
|
+
assert.equal(otherStoredSession?.session.sessionId, otherSession.session.sessionId);
|
|
89
|
+
assert.equal(otherStoredSession?.session.codex?.model, primaryModel);
|
|
90
|
+
|
|
91
|
+
const persisted = JSON.parse(await fs.readFile(fallbackStatePath, "utf8"));
|
|
92
|
+
persisted.resetAt = "2000-01-01T00:00:00.000Z";
|
|
93
|
+
await fs.writeFile(fallbackStatePath, `${JSON.stringify(persisted, null, 2)}\n`, "utf8");
|
|
94
|
+
calls.length = 0;
|
|
95
|
+
progress.length = 0;
|
|
96
|
+
failPrimaryWithUsageLimit = false;
|
|
97
|
+
bridge = createBridge();
|
|
98
|
+
const recovered = await bridge.routeMessage("test-bot", "test-chat", "after reset", async (response) => {
|
|
99
|
+
progress.push(response.output);
|
|
100
|
+
});
|
|
101
|
+
assert.deepEqual(calls.map((call) => call.model), [primaryModel]);
|
|
102
|
+
assert.equal(recovered[0]?.model, primaryModel);
|
|
103
|
+
assert.match(progress.join("\n"), /원래 모델로 복귀했습니다/);
|
|
104
|
+
await assert.rejects(fs.stat(fallbackStatePath), { code: "ENOENT" });
|
|
105
|
+
|
|
106
|
+
const fallbackService = new CodexUsageFallbackService(dataDir);
|
|
107
|
+
await fallbackService.activate(parseCodexUsageLimit("You've hit your usage limit.") ?? {});
|
|
108
|
+
assert.equal(await fallbackService.read(), undefined);
|
|
109
|
+
|
|
110
|
+
calls.length = 0;
|
|
111
|
+
failPrimaryWithUsageLimit = true;
|
|
112
|
+
failFallbackWithUsageLimit = true;
|
|
113
|
+
bridge = createBridge();
|
|
114
|
+
await assert.rejects(
|
|
115
|
+
bridge.routeMessage("test-bot", "test-chat", "fallback exhausted"),
|
|
116
|
+
/hit your usage limit/,
|
|
117
|
+
);
|
|
118
|
+
assert.deepEqual(calls.map((call) => call.model), [primaryModel, CODEX_USAGE_FALLBACK_MODEL]);
|
|
119
|
+
|
|
120
|
+
console.log(JSON.stringify({
|
|
121
|
+
ok: true,
|
|
122
|
+
detectedExactUsageError: true,
|
|
123
|
+
fallbackModel: CODEX_USAGE_FALLBACK_MODEL,
|
|
124
|
+
primaryModelPreserved: true,
|
|
125
|
+
fallbackPersistedAcrossBridgeRestart: true,
|
|
126
|
+
fallbackSharedAcrossSessions: true,
|
|
127
|
+
primaryRestoredAfterSuccessfulProbe: true,
|
|
128
|
+
fallbackAttemptLimit: 1,
|
|
129
|
+
}, null, 2));
|
|
130
|
+
} finally {
|
|
131
|
+
await fs.rm(root, { recursive: true, force: true });
|
|
132
|
+
}
|
|
@@ -104,6 +104,7 @@ let providerMode = "success";
|
|
|
104
104
|
let untaggedIntentCalls = 0;
|
|
105
105
|
let missingEvidenceCalls = 0;
|
|
106
106
|
let streamingFinalProgressCalls = 0;
|
|
107
|
+
let usageLimitCalls = 0;
|
|
107
108
|
let queueHoldStartedResolve;
|
|
108
109
|
let queueHoldReleaseResolve;
|
|
109
110
|
let queueHoldStartedPromise = Promise.resolve();
|
|
@@ -114,6 +115,19 @@ const provider = {
|
|
|
114
115
|
if (providerMode === "timeout") {
|
|
115
116
|
throw new Error("Codex timed out after 600s without returning a final reply.");
|
|
116
117
|
}
|
|
118
|
+
if (providerMode === "usage-limit") {
|
|
119
|
+
usageLimitCalls += 1;
|
|
120
|
+
if (request.model !== "gpt-5.3-codex-spark") {
|
|
121
|
+
throw new Error("You've hit your usage limit. Visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Aug 27th, 2099 3:52 AM.");
|
|
122
|
+
}
|
|
123
|
+
return {
|
|
124
|
+
provider: "codex",
|
|
125
|
+
sessionId: request.sessionId || "mock-thread",
|
|
126
|
+
publicSessionId: request.publicSessionId,
|
|
127
|
+
cwd: request.cwd,
|
|
128
|
+
output: "REPORT:result\nusage fallback completed with evidence: `fallback-test.log`",
|
|
129
|
+
};
|
|
130
|
+
}
|
|
117
131
|
if (providerMode === "untagged-intent") {
|
|
118
132
|
untaggedIntentCalls += 1;
|
|
119
133
|
return {
|
|
@@ -560,6 +574,36 @@ if (refreshedBotsCalls.length < 2) {
|
|
|
560
574
|
throw new Error("Bot refresh callback did not render the bot list again");
|
|
561
575
|
}
|
|
562
576
|
|
|
577
|
+
const usageProviderCallsBefore = providerCalls.length;
|
|
578
|
+
providerMode = "usage-limit";
|
|
579
|
+
await send("/batch start");
|
|
580
|
+
await send("usage limit fallback regression test");
|
|
581
|
+
await send("/batch send");
|
|
582
|
+
const usageCalls = await readTelegramCalls();
|
|
583
|
+
const usageProviderModels = providerCalls.slice(usageProviderCallsBefore).map((call) => call.model);
|
|
584
|
+
if (usageLimitCalls !== 2 || usageProviderModels.join(",") !== "gpt-5.6-terra,gpt-5.3-codex-spark") {
|
|
585
|
+
throw new Error(`Usage fallback did not make one primary and one Spark call: ${JSON.stringify(usageProviderModels)}`);
|
|
586
|
+
}
|
|
587
|
+
if (!usageCalls.some((call) => call.method === "sendMessage" && call.text.includes("임시 전환합니다"))) {
|
|
588
|
+
throw new Error("Usage fallback transition notice was not delivered to Telegram");
|
|
589
|
+
}
|
|
590
|
+
if (!usageCalls.some((call) =>
|
|
591
|
+
call.method === "sendMessage"
|
|
592
|
+
&& call.text.includes("[CODEX | gpt-5.3-codex-spark | S001]")
|
|
593
|
+
&& call.text.includes("usage fallback completed")
|
|
594
|
+
)) {
|
|
595
|
+
throw new Error("Usage fallback result did not identify the actual Spark execution model");
|
|
596
|
+
}
|
|
597
|
+
const usageFallbackState = JSON.parse(await fs.readFile(path.join(dataDir, "codex-usage-fallback.json"), "utf8"));
|
|
598
|
+
if (usageFallbackState.fallbackModel !== "gpt-5.3-codex-spark") {
|
|
599
|
+
throw new Error(`Usage fallback state was not persisted: ${JSON.stringify(usageFallbackState)}`);
|
|
600
|
+
}
|
|
601
|
+
const usageSessionState = JSON.parse(await fs.readFile(path.join(dataDir, "state.json"), "utf8"));
|
|
602
|
+
if (usageSessionState.sessions[session.sessionId]?.codex?.model !== "gpt-5.6-terra") {
|
|
603
|
+
throw new Error("Temporary usage fallback overwrote the session's primary model");
|
|
604
|
+
}
|
|
605
|
+
await fs.unlink(path.join(dataDir, "codex-usage-fallback.json"));
|
|
606
|
+
|
|
563
607
|
providerMode = "timeout";
|
|
564
608
|
await send("/batch start");
|
|
565
609
|
await send("timeout regression test");
|
|
@@ -752,6 +796,7 @@ console.log(JSON.stringify({
|
|
|
752
796
|
queueRemoveById: firstQueueId,
|
|
753
797
|
queueRemoveLatest: secondQueueId,
|
|
754
798
|
timeoutFinalMessage: true,
|
|
799
|
+
usageLimitFallback: true,
|
|
755
800
|
telegramSendMessages: evidenceCalls.filter((call) => call.method === "sendMessage").length,
|
|
756
801
|
}, null, 2));
|
|
757
802
|
|