appback-remoteagent 0.21.1 → 0.22.0

package/dist/services/bridge-service.js

@@ -3,8 +3,9 @@ import { randomBytes } from "node:crypto";
 import os from "node:os";
 import path from "node:path";
 import { stopSpawnedExecution } from "../adapters/windows-shell.js";
+import { CODEX_USAGE_FALLBACK_MODEL, CodexUsageFallbackService, parseCodexUsageLimit, } from "./codex-usage-fallback-service.js";
 const MODEL_PRESETS = {
-    codex: ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.6", "gpt-5.5", "gpt-5.4", "gpt-5.4-mini", "gpt-5.2", "gpt-5.1-codex-max"],
+    codex: ["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.6", "gpt-5.5", "gpt-5.4", "gpt-5.4-mini", "gpt-5.3-codex-spark", "gpt-5.2", "gpt-5.1-codex-max"],
     claude: ["sonnet", "opus", "haiku"],
 };
 const DEFAULT_CODEX_MODEL = "gpt-5.6-sol";
@@ -17,6 +18,7 @@ export class BridgeService {
     preferredStartMode;
     defaultCodexSandboxMode;
     sessionLocks = new Map();
+    codexUsageFallback;
     constructor(store, adapters, defaultWorkspace, workspaceRoot, isProviderInstalled, preferredStartMode, defaultCodexSandboxMode) {
         this.store = store;
         this.adapters = adapters;
@@ -25,6 +27,7 @@ export class BridgeService {
         this.isProviderInstalled = isProviderInstalled;
         this.preferredStartMode = preferredStartMode;
         this.defaultCodexSandboxMode = defaultCodexSandboxMode;
+        this.codexUsageFallback = new CodexUsageFallbackService(store.getDataDir());
     }
     async createSession(botId, chatId, cwd) {
         const provider = await this.resolveStartProvider();
@@ -490,7 +493,33 @@ export class BridgeService {
         for (const provider of providers) {
             const providerSession = this.ensureProviderSession(session, provider);
             this.ensureConfigured(provider);
-            const response = await this.adapters[provider].send({
+            const primaryModel = providerSession.model ?? this.defaultModelFor(provider);
+            const selection = provider === "codex"
+                ? await this.codexUsageFallback.selectExecutionModel(primaryModel)
+                : { model: primaryModel, recoveryProbe: false };
+            let executionModel = selection.model;
+            const forwardProgress = async (output, model) => {
+                const progressResponse = {
+                    provider,
+                    sessionId: providerSession.sessionId ?? session.sessionId,
+                    publicSessionId: session.publicId,
+                    model,
+                    cwd: providerSession.cwd,
+                    output,
+                };
+                await this.log({
+                    timestamp: new Date().toISOString(),
+                    remoteSessionId: session.sessionId,
+                    botId,
+                    chatId,
+                    provider,
+                    direction: "out",
+                    sessionId: providerSession.sessionId,
+                    text: output,
+                });
+                await onProgress?.(progressResponse);
+            };
+            const sendWithModel = (model) => this.adapters[provider].send({
                 botId,
                 chatId: chatId ?? requestSource,
                 remoteSessionId: session.sessionId,
@@ -498,32 +527,59 @@ export class BridgeService {
                 cwd: providerSession.cwd,
                 sessionId: providerSession.sessionId,
                 message,
-                model: providerSession.model,
+                model,
                 sandboxMode: providerSession.sandboxMode,
-                onProgress: onProgress
-                    ? async (output) => {
-                        const progressResponse = {
-                            provider,
-                            sessionId: providerSession.sessionId ?? session.sessionId,
-                            publicSessionId: session.publicId,
-                            model: providerSession.model ?? this.defaultModelFor(provider),
-                            cwd: providerSession.cwd,
-                            output,
-                        };
-                        await this.log({
-                            timestamp: new Date().toISOString(),
-                            remoteSessionId: session.sessionId,
-                            botId,
-                            chatId,
-                            provider,
-                            direction: "out",
-                            sessionId: providerSession.sessionId,
-                            text: output,
-                        });
-                        await onProgress(progressResponse);
-                    }
-                    : undefined,
+                onProgress: onProgress ? (output) => forwardProgress(output, model) : undefined,
             });
+            const reportHarnessNotice = async (output, model) => {
+                try {
+                    await forwardProgress(output, model);
+                }
+                catch (error) {
+                    if (error instanceof Error && error.name === "SilentTelegramAbort") {
+                        throw error;
+                    }
+                    const messageText = error instanceof Error ? error.message : String(error);
+                    console.warn(`[codex-usage-fallback] notice delivery failed for ${session.publicId}: ${messageText}`);
+                    await this.log({
+                        timestamp: new Date().toISOString(),
+                        remoteSessionId: session.sessionId,
+                        botId,
+                        chatId,
+                        provider: "system",
+                        direction: "system",
+                        sessionId: providerSession.sessionId,
+                        text: `Codex usage fallback notice delivery failed: ${messageText}`,
+                    });
+                }
+            };
+            let response;
+            try {
+                response = await sendWithModel(executionModel);
+            }
+            catch (error) {
+                const messageText = error instanceof Error ? error.message : String(error);
+                const usageLimit = provider === "codex" ? parseCodexUsageLimit(messageText) : undefined;
+                if (!usageLimit || executionModel === CODEX_USAGE_FALLBACK_MODEL) {
+                    throw error;
+                }
+                await this.codexUsageFallback.activate(usageLimit);
+                executionModel = CODEX_USAGE_FALLBACK_MODEL;
+                const resetText = usageLimit.resetAtLabel ? ` 초기화 예정: ${usageLimit.resetAtLabel}.` : "";
+                await reportHarnessNotice([
+                    "REPORT:progress",
+                    `기본 모델 ${primaryModel}의 사용 한도가 소진되어 이번 실행을 ${executionModel}로 임시 전환합니다.${resetText}`,
+                    "세션의 기본 모델 설정은 변경하지 않습니다.",
+                ].join("\n"), executionModel);
+                response = await sendWithModel(executionModel);
+            }
+            if (provider === "codex" && selection.recoveryProbe && executionModel === primaryModel) {
+                await this.codexUsageFallback.clear();
+                await reportHarnessNotice([
+                    "REPORT:progress",
+                    `사용 한도 초기화 후 기본 모델 ${primaryModel} 호출이 성공하여 원래 모델로 복귀했습니다.`,
+                ].join("\n"), primaryModel);
+            }
             const updatedProviderSession = {
                 ...providerSession,
                 cwd: response.cwd,
@@ -553,7 +609,7 @@ export class BridgeService {
             responses.push({
                 ...response,
                 publicSessionId: session.publicId,
-                model: providerSession.model ?? this.defaultModelFor(provider),
+                model: executionModel,
             });
         }
         return responses;

package/dist/services/codex-usage-fallback-service.js

@@ -0,0 +1,111 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { randomUUID } from "node:crypto";
+export const CODEX_USAGE_FALLBACK_MODEL = "gpt-5.3-codex-spark";
+export class CodexUsageFallbackService {
+    now;
+    statePath;
+    writeTail = Promise.resolve();
+    constructor(dataDir, now = () => new Date()) {
+        this.now = now;
+        this.statePath = path.join(dataDir, "codex-usage-fallback.json");
+    }
+    async selectExecutionModel(primaryModel) {
+        if (primaryModel === CODEX_USAGE_FALLBACK_MODEL) {
+            return { model: primaryModel, recoveryProbe: false };
+        }
+        const state = await this.read();
+        if (!state) {
+            return { model: primaryModel, recoveryProbe: false };
+        }
+        if (Date.parse(state.resetAt) > this.now().getTime()) {
+            return { model: state.fallbackModel, recoveryProbe: false };
+        }
+        return { model: primaryModel, recoveryProbe: true };
+    }
+    async activate(limit) {
+        if (!limit.resetAt || Date.parse(limit.resetAt) <= this.now().getTime()) {
+            return;
+        }
+        await this.write({
+            fallbackModel: CODEX_USAGE_FALLBACK_MODEL,
+            activatedAt: this.now().toISOString(),
+            resetAt: limit.resetAt,
+            resetAtLabel: limit.resetAtLabel,
+        });
+    }
+    async clear() {
+        await this.withWriteLock(async () => {
+            await fs.unlink(this.statePath).catch((error) => {
+                if (error.code !== "ENOENT") {
+                    throw error;
+                }
+            });
+        });
+    }
+    async read() {
+        const raw = await fs.readFile(this.statePath, "utf8").catch((error) => {
+            if (error.code === "ENOENT") {
+                return undefined;
+            }
+            throw error;
+        });
+        if (!raw) {
+            return undefined;
+        }
+        try {
+            const parsed = JSON.parse(raw);
+            if (parsed.fallbackModel !== CODEX_USAGE_FALLBACK_MODEL
+                || typeof parsed.activatedAt !== "string"
+                || typeof parsed.resetAt !== "string"
+                || !Number.isFinite(Date.parse(parsed.resetAt))) {
+                return undefined;
+            }
+            return parsed;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    async write(state) {
+        await this.withWriteLock(async () => {
+            await fs.mkdir(path.dirname(this.statePath), { recursive: true });
+            const temporaryPath = `${this.statePath}.${process.pid}.${randomUUID()}.tmp`;
+            await fs.writeFile(temporaryPath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 });
+            await fs.rename(temporaryPath, this.statePath);
+        });
+    }
+    async withWriteLock(task) {
+        const previous = this.writeTail;
+        let release;
+        this.writeTail = new Promise((resolve) => {
+            release = resolve;
+        });
+        await previous.catch(() => undefined);
+        try {
+            await task();
+        }
+        finally {
+            release();
+        }
+    }
+}
+export function parseCodexUsageLimit(message) {
+    if (!/you(?:'|’)ve hit your usage limit\b/i.test(message)) {
+        return undefined;
+    }
+    const resetMatch = /try again at\s+(.+?)(?:\.\s|\.?$|\n)/i.exec(message);
+    const resetAtLabel = resetMatch?.[1]?.trim();
+    if (!resetAtLabel) {
+        return {};
+    }
+    const normalized = resetAtLabel.replace(/(\d{1,2})(?:st|nd|rd|th)\b/gi, "$1");
+    const timestamp = Date.parse(normalized);
+    if (!Number.isFinite(timestamp)) {
+        return { resetAtLabel };
+    }
+    return {
+        resetAt: new Date(timestamp).toISOString(),
+        resetAtLabel,
+    };
+}

package/dist/store/file-store.js

@@ -51,6 +51,9 @@ export class FileStore {
             .filter((session) => sessionIds.has(session.sessionId))
             .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt));
     }
+    getDataDir() {
+        return this.dataDir;
+    }
     async listActiveSessionIds(botId) {
         const state = await this.readState();
         const sessionIds = Object.values(state.chats)

package/docs/ERROR_NORMALIZATION.md

@@ -21,6 +21,12 @@ RemoteAgent should never leak raw provider event payloads directly to Telegram u
 - `provider.timeout.final`
   - example: `Codex timed out after 600s without returning a final reply`
   - behavior: do not retry automatically; explain the exact configured timeout that killed the provider process
+- `provider.usage_limit.fallback`
+  - example: `You've hit your usage limit ... try again at Aug 27th, 2026 3:52 AM`
+  - behavior: preserve the session's primary model and retry the same request once with `gpt-5.3-codex-spark`
+  - scope: the persisted fallback window belongs to the machine's Codex account, not to an individual RemoteAgent session
+  - recovery: after the reported reset time, try the primary model on the next request and clear fallback state only after that call succeeds
+  - exclusions: model capacity, HTTP 429, Telegram rate limits, 503 responses, and transport errors must not activate this fallback
 
 ## Current terminal behavior
 
@@ -36,6 +42,14 @@ When a provider process reaches `COMMAND_TIMEOUT_MS`:
 - do not invent a cause such as context compaction, provider outage, or model quality unless the provider output explicitly says that
 - tell the operator to increase `/option timeout <seconds>` for long-running work
 
+When Codex reports an explicit account usage-limit error:
+
+- store the temporary state in `codex-usage-fallback.json`
+- do not modify the model stored in any session
+- use the same Codex thread id, workspace, prompt, and sandbox while overriding only the execution model
+- do not retry recursively if the fallback model also reports a usage-limit error
+- if the error does not contain a parseable future reset time, use Spark only for the current execution and probe the primary model again on the next request
+
 ## Maintenance rule
 
 When a new provider error shape appears:

package/docs/OPERATIONS.md

@@ -191,8 +191,8 @@ ssh au2223@192.168.0.30 'export PATH="/home/au2223/.local/bin:/home/au2223/.nvm/
 For server 26:
 
 ```bash
-ssh ospadmin@192.168.0.26 'export PATH="$HOME/.local/bin:$PATH"; npm explore -g appback-remoteagent -- npm run maintenance:disk -- report'
-ssh ospadmin@192.168.0.26 'export PATH="$HOME/.local/bin:$PATH"; npm explore -g appback-remoteagent -- npm run maintenance:disk -- prune-safe'
+ssh ospadmin@192.168.33.26 'export PATH="$HOME/.local/bin:$PATH"; npm explore -g appback-remoteagent -- npm run maintenance:disk -- report'
+ssh ospadmin@192.168.33.26 'export PATH="$HOME/.local/bin:$PATH"; npm explore -g appback-remoteagent -- npm run maintenance:disk -- prune-safe'
 ```
 
 ## Git workflow

package/docs/RELEASING.md

@@ -66,7 +66,8 @@ npm run release:publish
 
 The publish script performs:
 
-- clean working tree check
+- tracked working tree check
+- a clean staging snapshot from committed `HEAD`; unrelated untracked files are listed but excluded
 - `npm whoami`
 - `npm owner ls appback-remoteagent`
 - `npm run check`
@@ -86,6 +87,12 @@ Deploy to server 30:
 npm run release:deploy -- <version> 30
 ```
 
+Deploy to server 40:
+
+```bash
+npm run release:deploy -- <version> 40
+```
+
 Deploy to server 26:
 
 ```bash
@@ -108,6 +115,7 @@ The deploy script performs:
 
 - npm registry version check for `appback-remoteagent@<version>`
 - server 30 npm install, install hook, systemd restart, version/log verification
+- server 40 npm install, install hook, user-process restart, version/log verification
 - server 26 npm install, install hook, user-process restart, version/log verification
 
 ## 6. Verify
@@ -121,7 +129,7 @@ npm view appback-remoteagent version
 Server 30:
 
 ```bash
-ssh au2223@192.168.0.30 'bash -lc '"'"'
+ssh au2223@192.168.33.30 'bash -lc '"'"'
 export PATH="/home/au2223/.local/bin:/home/au2223/.nvm/versions/node/v22.22.0/bin:$PATH"
 systemctl is-active remoteagent
 node -p "require(\"/home/au2223/.nvm/versions/node/v22.22.0/lib/node_modules/appback-remoteagent/package.json\").version"
@@ -132,7 +140,18 @@ journalctl -u remoteagent --since "5 minutes ago" --no-pager | tail -80
 Server 26:
 
 ```bash
-ssh ospadmin@192.168.0.26 'bash -lc '"'"'
+ssh ospadmin@192.168.33.26 'bash -lc '"'"'
+npm list -g appback-remoteagent --depth=0
+pgrep -af "appback-remoteagent/dist/index.js"
+tail -80 ~/.remoteagent/logs/agent.log
+'"'"''
+```
+
+Server 40:
+
+```bash
+ssh appback@192.168.33.40 'bash -lc '"'"'
+export PATH="$HOME/.local/bin:$HOME/.nvm/versions/node/v22.23.2/bin:$PATH"
 npm list -g appback-remoteagent --depth=0
 pgrep -af "appback-remoteagent/dist/index.js"
 tail -80 ~/.remoteagent/logs/agent.log

package/docs/SERVER26_UBUNTU_MIGRATION_HANDOVER.md

@@ -0,0 +1,350 @@
+# Server 26 Ubuntu Migration Handover
+
+## 1. Purpose
+
+This document hands over the conversion of server 26 from the legacy
+RHEL/SQream/backup host into the primary production application server.
+
+Target role:
+
+- public address: `125.132.17.159`
+- internal address: `192.168.33.26`
+- SSH account before reinstall: `ospadmin`
+- proposed hostname after reinstall: `appback-prod`
+- proposed OS: a clean Ubuntu LTS installation
+- primary workload: production application containers
+- external disk: backup-only storage
+
+The internal address should remain `192.168.33.26`. There is no operational
+benefit in changing it again, and retaining it reduces changes to NAT,
+firewall, SSH, monitoring, and deployment configuration.
+
+This document does not authorize destructive work. Formatting or reinstalling
+the OS disk requires explicit user approval after the backup and restore gates
+below pass.
+
+## 2. Current Verified State
+
+Verified on `2026-08-13` through:
+
+```bash
+ssh ospadmin@192.168.33.26
+```
+
+### Host and network
+
+- current OS: Red Hat Enterprise Linux 8.9
+- current hostname: `scailium-node`
+- active interface: `enp0s31f6`
+- active address: `192.168.33.26/24`
+- current wired MAC: `1c:1b:0d:90:8b:ce`
+- former Wi-Fi profile: `DAONEWF5G`
+- former Wi-Fi interface: `wlp0s20f0u5`
+- former cloned Wi-Fi MAC: `00:15:5D:BC:FA:34`
+- DNS and outbound Telegram API access are currently working
+
+Confirm that the router maps public address `125.132.17.159` to this host
+before production cutover. Do not assume that the public address is configured
+directly on the Linux interface.
+
+### Disks
+
+| Device | Size | Filesystem | Current role |
+| --- | ---: | --- | --- |
+| `/dev/sdc` | 238.5GB | RHEL LVM/XFS | Current OS disk; intended Ubuntu install target |
+| `/dev/sda2` | 3.7TB | XFS, label `RA_BACKUP_4T` | Mounted at `/mnt/backup`; preserve |
+| `/dev/sdb1` | 1.8TB | NTFS | Currently unmounted; ownership and purpose must be confirmed |
+
+Current filesystem pressure:
+
+- `/`: 207GB total, 185GB used, 90%
+- `/home`: 23GB total, 14GB used, 63%
+- `/mnt/backup`: 3.7TB total, 73GB used, 2%
+
+Physically disconnect both `/dev/sda` and `/dev/sdb` before installing Ubuntu.
+Only `/dev/sdc` may be selected by the installer.
+
+## 3. Critical Backup Finding
+
+`/mnt/backup/pg` exists but is currently empty. The PostgreSQL dumps that must
+be preserved are still on the OS-side storage under `/data/cluster`.
+
+Verified dump inventory:
+
+| Source | Size | Dump count | Latest verified dump |
+| --- | ---: | ---: | --- |
+| `/data/cluster/appback-hub-backups` | 8.7GB | 141 | `2026-08-09 06:17 KST`, about 90MB |
+| `/data/cluster/grid-clash-backups` | 30GB | 122 | `2026-08-09 07:49 KST`, about 409MB |
+| `/data/cluster/damoa-backups` | 16GB | 4 | `2026-08-09 02:48 KST`, about 4.3GB |
+
+These dumps predate the current boot and are not yet confirmed as complete or
+restorable. File existence is not a restore test.
+
+The external disk currently contains media backup data instead:
+
+- `/mnt/backup/remoteagent/damoa-media/current`: about 2.3GB
+- `/mnt/backup/remoteagent/damoa-media/history`: about 44GB
+- `/mnt/backup/remoteagent/damoa-media/restore-tests`: empty
+
+### Mandatory copy stage
+
+Copy the database backup trees to the external disk without deleting either
+source or destination data:
+
+```bash
+sudo mkdir -p /mnt/backup/pg
+sudo rsync -aH --info=progress2 \
+  /data/cluster/appback-hub-backups/ \
+  /mnt/backup/pg/appback-hub-backups/
+sudo rsync -aH --info=progress2 \
+  /data/cluster/grid-clash-backups/ \
+  /mnt/backup/pg/grid-clash-backups/
+sudo rsync -aH --info=progress2 \
+  /data/cluster/damoa-backups/ \
+  /mnt/backup/pg/damoa-backups/
+```
+
+Generate a manifest after the copy:
+
+```bash
+cd /mnt/backup/pg
+find . -type f -name '*.dump' -print0 \
+  | sort -z \
+  | xargs -0 sha256sum \
+  > SHA256SUMS
+sha256sum -c SHA256SUMS
+```
+
+The copy stage is complete only when file counts, total sizes, and SHA-256
+verification match the source inventory.
+
+### Mandatory restore stage
+
+For each service, restore the newest selected dump into an isolated PostgreSQL
+instance and record:
+
+- PostgreSQL version used for restore
+- `pg_restore` exit status
+- restored database size
+- schema count and expected key tables
+- basic row-count checks agreed with the service owner
+- application health check against the restored database where possible
+
+Store the evidence under:
+
+```text
+/mnt/backup/pg/restore-evidence/<service>/<timestamp>/
+```
+
+Do not reinstall the OS until all required databases have a successful restore
+record. If the latest dump fails, stop and report it rather than silently using
+an older dump.
+
+## 4. Current RemoteAgent State
+
+RemoteAgent was repaired during the network investigation:
+
+- package: `appback-remoteagent@0.21.1`
+- runtime: user `systemd` service
+- unit: `~/.config/systemd/user/remoteagent.service`
+- state: active and enabled
+- linger: enabled for `ospadmin`
+- providers detected: Codex
+- configured Telegram bots observed:
+  - `@bot26pcbotbotbot_bot`
+  - `@ayeTemail_bot`
+
+Runtime data is under:
+
+```text
+/home/ospadmin/.remoteagent
+```
+
+The Secret Store contains operational credentials. Secret values must never be
+written into this repository or the handover report. Export it through the
+RemoteAgent secret migration procedure and store the encrypted archive on the
+external disk with mode `600`.
+
+Also preserve:
+
+- `~/.ssh/authorized_keys`
+- the current RemoteAgent `.env`
+- session/state data only if existing Telegram sessions must be resumed
+- `/home/ospadmin/crontab.before-network-fix-20260813-094632`
+
+Reinstall RemoteAgent from npm after Ubuntu is ready. Do not copy the old npm
+global package directory from RHEL.
+
+## 5. SQream and Scheduled Jobs
+
+SQream is not healthy and must not be treated as a running database.
+
+Actions already completed:
+
+- active SQream configuration IPs were changed from `192.168.0.26` to
+  `192.168.33.26`
+- overlapping market/strategy processes were terminated
+- the affected market cron lines were commented with `TEMP_NETWORK_FIX`
+- the original crontab was backed up at the path recorded above
+
+Current SQream state:
+
+- metadata server is reachable on port `3105`
+- server picker is reachable on port `3108`
+- active SQream workers: `0`
+- all five workers connect to metadata and then exit with:
+
+```text
+Error: mismatch in mac address validation Please contact SQream support.
+```
+
+The license appears tied to the former Wi-Fi MAC. Reissuing the license for the
+wired MAC would be required if SQream were retained. The target server role no
+longer requires SQream, so the recommended migration is to preserve only data
+that has an explicit owner and then omit SQream from the new Ubuntu build.
+
+Do not resume the paused market cron jobs on the current host. Decide separately
+whether those jobs are retired or moved to a dedicated analytics host.
+
+## 6. Recommended Migration Plan
+
+### Phase A: inventory and freeze
+
+1. Confirm the application services that will run on the new production host.
+2. Confirm whether all three PostgreSQL backup families must be retained.
+3. Confirm the owner and disposition of `/dev/sdb1`.
+4. Record router NAT, port forwarding, firewall, DNS, and TLS settings.
+5. Freeze configuration changes during the final backup window.
+
+Completion evidence:
+
+- signed-off service list
+- signed-off data retention list
+- exported network configuration
+- approved maintenance window
+
+### Phase B: preserve and test
+
+1. Copy all retained dumps to `/mnt/backup/pg`.
+2. Generate and verify checksums.
+3. Perform actual isolated restores.
+4. Back up SSH access, encrypted RemoteAgent secrets, and required configuration.
+5. Copy the migration evidence to a second machine or storage target.
+
+Completion evidence:
+
+- successful checksum log
+- successful restore evidence for every retained database
+- two independent copies of critical credentials and restore evidence
+
+### Phase C: clean Ubuntu installation
+
+1. Shut down the host.
+2. Physically disconnect the 3.7TB external disk and the 1.8TB NTFS disk.
+3. Install a mature Ubuntu LTS release on `/dev/sdc` only.
+4. Set hostname `appback-prod`.
+5. Configure static address `192.168.33.26/24` and gateway
+   `192.168.33.1`.
+6. Apply security updates and configure time synchronization.
+7. Restore SSH access and verify a second administrative session before
+   closing the installation console.
+
+The installer record must include the selected target disk and partition map.
+
+### Phase D: production runtime
+
+Recommended filesystem ownership:
+
+```text
+/opt/appback/compose     deployment definitions
+/srv/appback/data       persistent application data
+/srv/appback/logs       application logs
+/mnt/backup             external backup disk
+```
+
+Install only the required runtime components:
+
+- Docker Engine and Compose plugin
+- reverse proxy such as Nginx or Caddy
+- production application stack
+- monitoring and log rotation
+- RemoteAgent from the published npm package, if it remains required on 26
+
+Expose only HTTP/HTTPS publicly. Keep SSH and database ports restricted to the
+internal network or VPN. Verify whether `125.132.17.159` is routed by NAT before
+changing host firewall rules.
+
+### Phase E: restore and cutover
+
+1. Reconnect the external disk after Ubuntu is fully installed.
+2. Mount it by UUID at `/mnt/backup`; do not reformat it.
+3. Restore application databases from the verified backup set.
+4. Deploy applications from versioned images/configuration.
+5. Run service, database, TLS, upload, and external access checks.
+6. Enable scheduled backups only after a manual backup and restore test passes.
+7. Change DNS/NAT only during the approved cutover window.
+
+## 7. Acceptance Criteria
+
+The migration is complete only when all of the following are recorded:
+
+- Ubuntu boots with the intended hostname and static internal address
+- SSH works from an authorized internal host
+- only approved public ports are exposed
+- each production container has a versioned image and restart policy
+- each required database was restored from a verified dump
+- application health and representative user flows pass
+- backup writes to the external disk
+- a post-install restore test succeeds
+- disk, CPU, memory, container, and certificate monitoring is active
+- RemoteAgent, if installed, runs as exactly one supervised service
+- no SQream or legacy market cron process remains unless separately approved
+
+## 8. Stop Conditions
+
+Stop and report before any destructive step if any of these is true:
+
+- `/mnt/backup/pg` has not been populated and checksum-verified
+- a required database has no successful restore test
+- the external and NTFS disks cannot be positively distinguished from the OS disk
+- public-IP NAT ownership is unclear
+- required credentials or SSH recovery access have only one copy
+- the application inventory or retention decision is incomplete
+- an unexpected active workload is found on server 26
+
+## 9. Rollback
+
+Before OS installation, rollback means leaving RHEL unchanged and keeping the
+paused cron jobs paused.
+
+After OS installation, rollback cannot restore the old RHEL installation unless
+a full disk image was created. The operational rollback is therefore:
+
+1. keep the prior production server serving traffic
+2. restore the verified database backup to the prior or replacement host
+3. revert DNS/NAT to the prior production endpoint
+4. collect Ubuntu deployment logs before attempting another cutover
+
+Do not designate server 26 as the sole production host until this rollback path
+has been tested.
+
+## 10. First Commands for the Next Agent
+
+Use these read-only checks before changing anything:
+
+```bash
+ssh ospadmin@192.168.33.26
+hostnamectl
+ip -br addr
+lsblk -o NAME,SIZE,FSTYPE,LABEL,UUID,MOUNTPOINT
+df -hT / /home /mnt/backup
+systemctl --user status remoteagent --no-pager
+pgrep -af 'sqream|remoteagent|run_kr_strategy|run_toss_kr'
+crontab -l
+find /data/cluster -type f -name '*.dump' -printf '%TY-%Tm-%Td %TH:%TM %s %p\n' | sort | tail
+find /mnt/backup/pg -type f -name '*.dump' -printf '%TY-%Tm-%Td %TH:%TM %s %p\n' | sort | tail
+```
+
+The next agent should first report the current state and the exact phase it is
+starting. No formatting, deletion, package removal, or cron reactivation should
+occur merely from reading this document.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "appback-remoteagent",
-  "version": "0.21.1",
+  "version": "0.22.0",
   "description": "Personal installable session server for continuing local AI work across PC and Telegram",
   "license": "MIT",
   "type": "module",
@@ -20,6 +20,7 @@
     "check": "tsc --noEmit -p tsconfig.json",
     "selftest:telegram": "npm run build && node scripts/selftest-telegram-update.mjs",
     "selftest:codex-stream": "npm run build && node scripts/selftest-codex-stream.mjs",
+    "selftest:model-fallback": "npm run build && node scripts/selftest-model-fallback.mjs",
     "selftest:cli": "npm run build && node scripts/selftest-cli.mjs",
     "prepare": "npm run build",
     "prepublishOnly": "node scripts/prepublish-guard.mjs",

package/scripts/release-deploy.sh

@@ -2,7 +2,7 @@
 set -euo pipefail
 
 usage() {
-  echo "Usage: npm run release:deploy -- <version> <30|26|all>" >&2
+  echo "Usage: npm run release:deploy -- <version> <30|40|26|all>" >&2
   echo "Example: npm run release:deploy -- 0.15.5 all" >&2
 }
 
@@ -20,7 +20,7 @@ if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
 fi
 
 case "$TARGET" in
-  30|26|all)
+  30|40|26|all)
     ;;
   *)
     usage
@@ -36,7 +36,7 @@ if [[ "$REGISTRY_VERSION" != "$VERSION" ]]; then
 fi
 
 deploy_30() {
-  ssh au2223@192.168.0.30 "VERSION=$VERSION bash -s" <<'REMOTE'
+  ssh au2223@192.168.33.30 "VERSION=$VERSION bash -s" <<'REMOTE'
 set -euo pipefail
 export PATH="/home/au2223/.local/bin:/home/au2223/.nvm/versions/node/v22.22.0/bin:$PATH"
 node - <<'NODE'
@@ -63,7 +63,7 @@ REMOTE
 }
 
 deploy_26() {
-  ssh ospadmin@192.168.0.26 "VERSION=$VERSION bash -s" <<'REMOTE'
+  ssh ospadmin@192.168.33.26 "VERSION=$VERSION bash -s" <<'REMOTE'
 set -euo pipefail
 export PATH="$HOME/.local/bin:$PATH"
 node - <<'NODE'
@@ -93,15 +93,50 @@ tail -80 ~/.remoteagent/logs/agent.log
 REMOTE
 }
 
+deploy_40() {
+  ssh appback@192.168.33.40 "VERSION=$VERSION bash -s" <<'REMOTE'
+set -euo pipefail
+export PATH="$HOME/.local/bin:$HOME/.nvm/versions/node/v22.23.2/bin:$PATH"
+node - <<'NODE'
+const fs = require("fs");
+const path = `${process.env.HOME}/.remoteagent/bot-polling-state.json`;
+if (fs.existsSync(path)) {
+  const state = JSON.parse(fs.readFileSync(path, "utf8"));
+  const running = Object.values(state.bots || {})
+    .filter((bot) => Array.isArray(bot.runningSessionIds) && bot.runningSessionIds.length > 0)
+    .map((bot) => `${bot.username || bot.botId}: ${bot.runningSessionIds.join(", ")}`);
+  if (running.length > 0) {
+    console.error("RemoteAgent has active provider work. Retry deploy after it finishes:");
+    for (const item of running) console.error(`- ${item}`);
+    process.exit(2);
+  }
+}
+NODE
+npm install -g "appback-remoteagent@$VERSION"
+remoteagent-install
+~/.remoteagent/stop-remoteagent.sh || true
+sleep 2
+~/.remoteagent/start-remoteagent.sh
+sleep 5
+npm list -g appback-remoteagent --depth=0
+pgrep -af 'appback-remoteagent/dist/index.js'
+tail -80 ~/.remoteagent/logs/agent.log
+REMOTE
+}
+
 case "$TARGET" in
   30)
     deploy_30
     ;;
+  40)
+    deploy_40
+    ;;
   26)
     deploy_26
     ;;
   all)
     deploy_30
+    deploy_40
     deploy_26
     ;;
 esac

package/scripts/release-publish.sh

@@ -12,12 +12,19 @@ if [[ "$PACKAGE_NAME" != "appback-remoteagent" ]]; then
   exit 1
 fi
 
-if [[ -n "$(git status --porcelain)" ]]; then
-  echo "Working tree is not clean. Commit and push before publishing." >&2
-  git status --short >&2
+if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then
+  echo "Tracked files are not clean. Commit and push before publishing." >&2
+  git status --short --untracked-files=no >&2
   exit 1
 fi
 
+UNTRACKED="$(git ls-files --others --exclude-standard)"
+if [[ -n "$UNTRACKED" ]]; then
+  echo "Untracked files are excluded from the release snapshot:"
+  printf '%s\n' "$UNTRACKED"
+  echo
+fi
+
 echo "Publishing $PACKAGE_NAME@$VERSION"
 echo
 echo "npm identity:"
@@ -30,12 +37,20 @@ echo
 npm run check
 npm run build
 
-TARBALL="$(npm pack --silent)"
+STAGING_DIR="$(mktemp -d)"
 cleanup() {
-  rm -f "$TARBALL"
+  rm -rf "$STAGING_DIR"
 }
 trap cleanup EXIT
 
+git archive --format=tar HEAD | tar -xf - -C "$STAGING_DIR"
+cd "$STAGING_DIR"
+npm ci --ignore-scripts
+npm run check
+npm run build
+
+TARBALL="$(npm pack --silent)"
+
 echo
 echo "Publishing tarball: $TARBALL"
 REMOTEAGENT_PUBLISH_GUARD_OK=1 npm publish "$TARBALL" --access public

package/scripts/selftest-model-fallback.mjs

@@ -0,0 +1,132 @@
+#!/usr/bin/env node
+
+import assert from "node:assert/strict";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+import { BridgeService } from "../dist/services/bridge-service.js";
+import {
+  CODEX_USAGE_FALLBACK_MODEL,
+  CodexUsageFallbackService,
+  parseCodexUsageLimit,
+} from "../dist/services/codex-usage-fallback-service.js";
+import { FileStore } from "../dist/store/file-store.js";
+
+const root = await fs.mkdtemp(path.join(os.tmpdir(), "remoteagent-model-fallback-"));
+const dataDir = path.join(root, "data");
+const workspaceRoot = path.join(root, "workspaces");
+const defaultWorkspace = path.join(root, "default-workspace");
+const fallbackStatePath = path.join(dataDir, "codex-usage-fallback.json");
+const primaryModel = "gpt-5.6-sol";
+const usageError = "You've hit your usage limit. Visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Aug 27th, 2099 3:52 AM.";
+
+try {
+  await fs.mkdir(defaultWorkspace, { recursive: true });
+  const parsed = parseCodexUsageLimit(usageError);
+  assert.ok(parsed?.resetAt);
+  assert.equal(parsed.resetAtLabel, "Aug 27th, 2099 3:52 AM");
+  assert.equal(parseCodexUsageLimit("Selected model is at capacity."), undefined);
+  assert.equal(parseCodexUsageLimit("Too Many Requests: retry after 5"), undefined);
+
+  const store = new FileStore(dataDir, "codex");
+  await store.init();
+  const calls = [];
+  const progress = [];
+  let failPrimaryWithUsageLimit = true;
+  let failFallbackWithUsageLimit = false;
+  const provider = {
+    async send(request) {
+      calls.push({ model: request.model, message: request.message, sessionId: request.sessionId });
+      if (request.model === primaryModel && failPrimaryWithUsageLimit) {
+        throw new Error(usageError);
+      }
+      if (request.model === CODEX_USAGE_FALLBACK_MODEL && failFallbackWithUsageLimit) {
+        throw new Error("You've hit your usage limit. Try again at Aug 28th, 2099 1:00 AM.");
+      }
+      return {
+        provider: "codex",
+        sessionId: request.sessionId || "thread-model-fallback",
+        cwd: request.cwd,
+        output: "REPORT:result\nmodel fallback self-test completed",
+      };
+    },
+  };
+  const createBridge = () => new BridgeService(
+    store,
+    { codex: provider },
+    defaultWorkspace,
+    workspaceRoot,
+    (providerName) => providerName === "codex",
+    "codex",
+    "workspace-write",
+  );
+
+  let bridge = createBridge();
+  const started = await bridge.startSession("test-bot", "test-chat", "codex");
+  const originalSessionId = started.session.sessionId;
+  const first = await bridge.routeMessage("test-bot", "test-chat", "first request", async (response) => {
+    progress.push(response.output);
+  });
+  assert.deepEqual(calls.map((call) => call.model), [primaryModel, CODEX_USAGE_FALLBACK_MODEL]);
+  assert.deepEqual(calls.map((call) => call.message), ["first request", "first request"]);
+  assert.equal(first[0]?.model, CODEX_USAGE_FALLBACK_MODEL);
+  assert.match(progress.join("\n"), /임시 전환합니다/);
+
+  const afterFallback = await store.getChatSession("test-bot", "test-chat");
+  assert.equal(afterFallback?.session.sessionId, originalSessionId);
+  assert.equal(afterFallback?.session.codex?.model, primaryModel);
+  assert.ok(await fs.stat(fallbackStatePath));
+
+  calls.length = 0;
+  bridge = createBridge();
+  const otherSession = await bridge.startSession("other-bot", "other-chat", "codex");
+  await bridge.routeMessage("other-bot", "other-chat", "second request");
+  assert.deepEqual(calls.map((call) => call.model), [CODEX_USAGE_FALLBACK_MODEL]);
+  assert.equal(calls[0]?.sessionId, undefined);
+  const otherStoredSession = await store.getChatSession("other-bot", "other-chat");
+  assert.equal(otherStoredSession?.session.sessionId, otherSession.session.sessionId);
+  assert.equal(otherStoredSession?.session.codex?.model, primaryModel);
+
+  const persisted = JSON.parse(await fs.readFile(fallbackStatePath, "utf8"));
+  persisted.resetAt = "2000-01-01T00:00:00.000Z";
+  await fs.writeFile(fallbackStatePath, `${JSON.stringify(persisted, null, 2)}\n`, "utf8");
+  calls.length = 0;
+  progress.length = 0;
+  failPrimaryWithUsageLimit = false;
+  bridge = createBridge();
+  const recovered = await bridge.routeMessage("test-bot", "test-chat", "after reset", async (response) => {
+    progress.push(response.output);
+  });
+  assert.deepEqual(calls.map((call) => call.model), [primaryModel]);
+  assert.equal(recovered[0]?.model, primaryModel);
+  assert.match(progress.join("\n"), /원래 모델로 복귀했습니다/);
+  await assert.rejects(fs.stat(fallbackStatePath), { code: "ENOENT" });
+
+  const fallbackService = new CodexUsageFallbackService(dataDir);
+  await fallbackService.activate(parseCodexUsageLimit("You've hit your usage limit.") ?? {});
+  assert.equal(await fallbackService.read(), undefined);
+
+  calls.length = 0;
+  failPrimaryWithUsageLimit = true;
+  failFallbackWithUsageLimit = true;
+  bridge = createBridge();
+  await assert.rejects(
+    bridge.routeMessage("test-bot", "test-chat", "fallback exhausted"),
+    /hit your usage limit/,
+  );
+  assert.deepEqual(calls.map((call) => call.model), [primaryModel, CODEX_USAGE_FALLBACK_MODEL]);
+
+  console.log(JSON.stringify({
+    ok: true,
+    detectedExactUsageError: true,
+    fallbackModel: CODEX_USAGE_FALLBACK_MODEL,
+    primaryModelPreserved: true,
+    fallbackPersistedAcrossBridgeRestart: true,
+    fallbackSharedAcrossSessions: true,
+    primaryRestoredAfterSuccessfulProbe: true,
+    fallbackAttemptLimit: 1,
+  }, null, 2));
+} finally {
+  await fs.rm(root, { recursive: true, force: true });
+}

package/scripts/selftest-telegram-update.mjs

@@ -104,6 +104,7 @@ let providerMode = "success";
 let untaggedIntentCalls = 0;
 let missingEvidenceCalls = 0;
 let streamingFinalProgressCalls = 0;
+let usageLimitCalls = 0;
 let queueHoldStartedResolve;
 let queueHoldReleaseResolve;
 let queueHoldStartedPromise = Promise.resolve();
@@ -114,6 +115,19 @@ const provider = {
     if (providerMode === "timeout") {
       throw new Error("Codex timed out after 600s without returning a final reply.");
     }
+    if (providerMode === "usage-limit") {
+      usageLimitCalls += 1;
+      if (request.model !== "gpt-5.3-codex-spark") {
+        throw new Error("You've hit your usage limit. Visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Aug 27th, 2099 3:52 AM.");
+      }
+      return {
+        provider: "codex",
+        sessionId: request.sessionId || "mock-thread",
+        publicSessionId: request.publicSessionId,
+        cwd: request.cwd,
+        output: "REPORT:result\nusage fallback completed with evidence: `fallback-test.log`",
+      };
+    }
     if (providerMode === "untagged-intent") {
       untaggedIntentCalls += 1;
       return {
@@ -560,6 +574,36 @@ if (refreshedBotsCalls.length < 2) {
   throw new Error("Bot refresh callback did not render the bot list again");
 }
 
+const usageProviderCallsBefore = providerCalls.length;
+providerMode = "usage-limit";
+await send("/batch start");
+await send("usage limit fallback regression test");
+await send("/batch send");
+const usageCalls = await readTelegramCalls();
+const usageProviderModels = providerCalls.slice(usageProviderCallsBefore).map((call) => call.model);
+if (usageLimitCalls !== 2 || usageProviderModels.join(",") !== "gpt-5.6-terra,gpt-5.3-codex-spark") {
+  throw new Error(`Usage fallback did not make one primary and one Spark call: ${JSON.stringify(usageProviderModels)}`);
+}
+if (!usageCalls.some((call) => call.method === "sendMessage" && call.text.includes("임시 전환합니다"))) {
+  throw new Error("Usage fallback transition notice was not delivered to Telegram");
+}
+if (!usageCalls.some((call) =>
+  call.method === "sendMessage"
+  && call.text.includes("[CODEX | gpt-5.3-codex-spark | S001]")
+  && call.text.includes("usage fallback completed")
+)) {
+  throw new Error("Usage fallback result did not identify the actual Spark execution model");
+}
+const usageFallbackState = JSON.parse(await fs.readFile(path.join(dataDir, "codex-usage-fallback.json"), "utf8"));
+if (usageFallbackState.fallbackModel !== "gpt-5.3-codex-spark") {
+  throw new Error(`Usage fallback state was not persisted: ${JSON.stringify(usageFallbackState)}`);
+}
+const usageSessionState = JSON.parse(await fs.readFile(path.join(dataDir, "state.json"), "utf8"));
+if (usageSessionState.sessions[session.sessionId]?.codex?.model !== "gpt-5.6-terra") {
+  throw new Error("Temporary usage fallback overwrote the session's primary model");
+}
+await fs.unlink(path.join(dataDir, "codex-usage-fallback.json"));
+
 providerMode = "timeout";
 await send("/batch start");
 await send("timeout regression test");
@@ -752,6 +796,7 @@ console.log(JSON.stringify({
   queueRemoveById: firstQueueId,
   queueRemoveLatest: secondQueueId,
   timeoutFinalMessage: true,
+  usageLimitFallback: true,
   telegramSendMessages: evidenceCalls.filter((call) => call.method === "sendMessage").length,
 }, null, 2));