@remnic/server 9.35.3 → 9.35.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +38 -26
- package/dist/index.js +191 -156
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -16,6 +16,37 @@ declare function envOverrides(): Partial<ServerConfig["server"]> & {
|
|
|
16
16
|
remnic?: Record<string, unknown>;
|
|
17
17
|
};
|
|
18
18
|
|
|
19
|
+
interface StartupReadinessState {
|
|
20
|
+
ready: boolean;
|
|
21
|
+
warmupAttempts: number;
|
|
22
|
+
lastError?: string | null;
|
|
23
|
+
/** True when the gate opened before search warm-up completed (issue #2215). */
|
|
24
|
+
degraded?: boolean;
|
|
25
|
+
}
|
|
26
|
+
type StartupReadinessOutcome = "warmed" | "cancelled" | "overridden" | "search-disabled";
|
|
27
|
+
declare function runStartupSearchWarmup(options: {
|
|
28
|
+
signal: AbortSignal;
|
|
29
|
+
isAvailable: () => boolean;
|
|
30
|
+
search: (onDegradation: (code: string) => void) => Promise<unknown>;
|
|
31
|
+
}): Promise<void>;
|
|
32
|
+
declare function completeStartupReadiness(options: {
|
|
33
|
+
deferredReady: Promise<void>;
|
|
34
|
+
warmup: (signal: AbortSignal) => Promise<unknown>;
|
|
35
|
+
prepareWarmup?: (signal: AbortSignal) => Promise<boolean>;
|
|
36
|
+
state: StartupReadinessState;
|
|
37
|
+
timeoutMs?: number;
|
|
38
|
+
retryIntervalMs?: number;
|
|
39
|
+
/** Failed attempts before the gate opens degraded; 0 disables (strict gate). */
|
|
40
|
+
degradedAfterAttempts?: number;
|
|
41
|
+
override?: boolean;
|
|
42
|
+
skipWarmup?: () => boolean;
|
|
43
|
+
openGate: () => void;
|
|
44
|
+
shutdownSignal?: AbortSignal;
|
|
45
|
+
warn?: (message: string) => void;
|
|
46
|
+
info?: (message: string) => void;
|
|
47
|
+
error?: (message: string) => void;
|
|
48
|
+
}): Promise<StartupReadinessOutcome>;
|
|
49
|
+
|
|
19
50
|
/**
|
|
20
51
|
* @remnic/server
|
|
21
52
|
*
|
|
@@ -46,6 +77,12 @@ interface ServerConfig {
|
|
|
46
77
|
adminConsolePublicDir?: string;
|
|
47
78
|
adminConsolePrefillToken?: boolean;
|
|
48
79
|
readinessOverride?: boolean;
|
|
80
|
+
/**
|
|
81
|
+
* Failed search warm-up attempts before the init gate opens in degraded
|
|
82
|
+
* mode (issue #2215). 0 keeps the strict gate (health stays 503 until
|
|
83
|
+
* warm-up completes).
|
|
84
|
+
*/
|
|
85
|
+
readinessDegradedAfterAttempts?: unknown;
|
|
49
86
|
/** OAuth authorization-server facade for ChatGPT dev-mode apps (parsed by oauth.ts). */
|
|
50
87
|
oauth?: unknown;
|
|
51
88
|
};
|
|
@@ -62,6 +99,7 @@ interface ParsedServerConfig {
|
|
|
62
99
|
adminConsolePublicDir?: string;
|
|
63
100
|
adminConsolePrefillToken: boolean;
|
|
64
101
|
readinessOverride: boolean;
|
|
102
|
+
readinessDegradedAfterAttempts: number;
|
|
65
103
|
}
|
|
66
104
|
declare function parseServerConfig(raw: Partial<ServerConfig["server"]>, options?: {
|
|
67
105
|
portSource?: string;
|
|
@@ -69,32 +107,6 @@ declare function parseServerConfig(raw: Partial<ServerConfig["server"]>, options
|
|
|
69
107
|
declare function loadConfigFile(configPath: string): ServerConfig;
|
|
70
108
|
declare function mergeRemnicConfigForServer(fileRemnic: Record<string, unknown>, envRemnic: Record<string, unknown> | undefined): Record<string, unknown>;
|
|
71
109
|
declare function createAdminControls(configPath: string, config: PluginConfig, serverConfig: ParsedServerConfig): RemnicAdminControls;
|
|
72
|
-
interface StartupReadinessState {
|
|
73
|
-
ready: boolean;
|
|
74
|
-
warmupAttempts: number;
|
|
75
|
-
lastError?: string | null;
|
|
76
|
-
}
|
|
77
|
-
type StartupReadinessOutcome = "warmed" | "cancelled" | "overridden" | "search-disabled";
|
|
78
|
-
declare function runStartupSearchWarmup(options: {
|
|
79
|
-
signal: AbortSignal;
|
|
80
|
-
isAvailable: () => boolean;
|
|
81
|
-
search: (onDegradation: (code: string) => void) => Promise<unknown>;
|
|
82
|
-
}): Promise<void>;
|
|
83
|
-
declare function completeStartupReadiness(options: {
|
|
84
|
-
deferredReady: Promise<void>;
|
|
85
|
-
warmup: (signal: AbortSignal) => Promise<unknown>;
|
|
86
|
-
prepareWarmup?: (signal: AbortSignal) => Promise<boolean>;
|
|
87
|
-
state: StartupReadinessState;
|
|
88
|
-
timeoutMs?: number;
|
|
89
|
-
retryIntervalMs?: number;
|
|
90
|
-
override?: boolean;
|
|
91
|
-
skipWarmup?: () => boolean;
|
|
92
|
-
openGate: () => void;
|
|
93
|
-
shutdownSignal?: AbortSignal;
|
|
94
|
-
warn?: (message: string) => void;
|
|
95
|
-
info?: (message: string) => void;
|
|
96
|
-
error?: (message: string) => void;
|
|
97
|
-
}): Promise<StartupReadinessOutcome>;
|
|
98
110
|
interface ServerResult {
|
|
99
111
|
config: PluginConfig;
|
|
100
112
|
service: EngramAccessService;
|
package/dist/index.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// src/index.ts
|
|
4
4
|
import fs from "fs";
|
|
5
5
|
import path from "path";
|
|
6
|
-
import { parseConfig, isOpenaiApiKeyDisabled, resolveRemnicConfigRecord, Orchestrator, EngramAccessService, EngramAccessHttpServer, initLogger, log as
|
|
6
|
+
import { parseConfig, isOpenaiApiKeyDisabled, resolveRemnicConfigRecord, Orchestrator, EngramAccessService, EngramAccessHttpServer, initLogger, log as log3, getAllValidTokens, getAllValidTokenEntriesCached, expandTildePath } from "@remnic/core";
|
|
7
7
|
import { probeBetterSqlite3Driver } from "@remnic/core/runtime/better-sqlite";
|
|
8
8
|
|
|
9
9
|
// src/oauth.ts
|
|
@@ -696,6 +696,10 @@ function envOverrides() {
|
|
|
696
696
|
const adminConsolePublicDir = readCompatEnv("REMNIC_ADMIN_CONSOLE_PUBLIC_DIR", "ENGRAM_ADMIN_CONSOLE_PUBLIC_DIR");
|
|
697
697
|
const adminConsolePrefillToken = readCompatEnv("REMNIC_ADMIN_CONSOLE_PREFILL_TOKEN", "ENGRAM_ADMIN_CONSOLE_PREFILL_TOKEN");
|
|
698
698
|
const readinessOverride = process.env.REMNIC_READY_OVERRIDE;
|
|
699
|
+
const readinessDegradedAfterAttempts = readCompatEnv(
|
|
700
|
+
"REMNIC_READY_DEGRADED_AFTER_ATTEMPTS",
|
|
701
|
+
"ENGRAM_READY_DEGRADED_AFTER_ATTEMPTS"
|
|
702
|
+
);
|
|
699
703
|
const writeRateLimitMaxRequests = readCompatEnv(
|
|
700
704
|
"REMNIC_WRITE_RATE_LIMIT_MAX_REQUESTS",
|
|
701
705
|
"ENGRAM_WRITE_RATE_LIMIT_MAX_REQUESTS"
|
|
@@ -711,6 +715,7 @@ function envOverrides() {
|
|
|
711
715
|
if (adminConsolePublicDir) overrides.adminConsolePublicDir = adminConsolePublicDir;
|
|
712
716
|
if (adminConsolePrefillToken) overrides.adminConsolePrefillToken = adminConsolePrefillToken;
|
|
713
717
|
if (readinessOverride !== void 0) overrides.readinessOverride = readinessOverride;
|
|
718
|
+
if (readinessDegradedAfterAttempts !== void 0) overrides.readinessDegradedAfterAttempts = readinessDegradedAfterAttempts;
|
|
714
719
|
if (writeRateLimitMaxRequests !== void 0) overrides.writeRateLimitMaxRequests = writeRateLimitMaxRequests;
|
|
715
720
|
if (writeRateLimitWindowMs !== void 0) overrides.writeRateLimitWindowMs = writeRateLimitWindowMs;
|
|
716
721
|
if (process.env.OPENAI_API_KEY) remnic.openaiApiKey = process.env.OPENAI_API_KEY;
|
|
@@ -719,6 +724,161 @@ function envOverrides() {
|
|
|
719
724
|
return { ...overrides, ...Object.keys(remnic).length > 0 ? { remnic } : {} };
|
|
720
725
|
}
|
|
721
726
|
|
|
727
|
+
// src/startup-readiness.ts
|
|
728
|
+
import { log as log2 } from "@remnic/core";
|
|
729
|
+
function abortableDelay(ms, signal) {
|
|
730
|
+
if (signal.aborted) return Promise.resolve();
|
|
731
|
+
const { promise, resolve } = Promise.withResolvers();
|
|
732
|
+
const timer = setTimeout(resolve, ms);
|
|
733
|
+
const onAbort = () => {
|
|
734
|
+
clearTimeout(timer);
|
|
735
|
+
resolve();
|
|
736
|
+
};
|
|
737
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
738
|
+
return promise.finally(() => signal.removeEventListener("abort", onAbort));
|
|
739
|
+
}
|
|
740
|
+
var STARTUP_WARMUP_TIMEOUT_MS = 2e4;
|
|
741
|
+
var STARTUP_WARMUP_RETRY_INTERVAL_MS = 3e4;
|
|
742
|
+
var STARTUP_DEGRADED_AFTER_ATTEMPTS = 3;
|
|
743
|
+
var StartupWarmupDegradationError = class extends Error {
|
|
744
|
+
constructor(code) {
|
|
745
|
+
super(`startup search degraded: ${code}`);
|
|
746
|
+
this.name = "StartupWarmupDegradationError";
|
|
747
|
+
}
|
|
748
|
+
};
|
|
749
|
+
var StartupSyncPendingError = class extends Error {
|
|
750
|
+
constructor() {
|
|
751
|
+
super("startup search sync is not complete");
|
|
752
|
+
this.name = "StartupSyncPendingError";
|
|
753
|
+
}
|
|
754
|
+
};
|
|
755
|
+
async function runStartupSearchWarmup(options) {
|
|
756
|
+
let degradationCode;
|
|
757
|
+
await options.search((code) => {
|
|
758
|
+
degradationCode = code;
|
|
759
|
+
});
|
|
760
|
+
if (options.signal.aborted) return;
|
|
761
|
+
if (degradationCode) throw new StartupWarmupDegradationError(degradationCode);
|
|
762
|
+
if (!options.isAvailable()) {
|
|
763
|
+
throw new StartupWarmupDegradationError("backend_unavailable");
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
async function completeStartupReadiness(options) {
|
|
767
|
+
const timeoutMs = options.timeoutMs ?? STARTUP_WARMUP_TIMEOUT_MS;
|
|
768
|
+
const retryIntervalMs = options.retryIntervalMs ?? STARTUP_WARMUP_RETRY_INTERVAL_MS;
|
|
769
|
+
const degradedAfterAttempts = options.degradedAfterAttempts ?? STARTUP_DEGRADED_AFTER_ATTEMPTS;
|
|
770
|
+
const warn = options.warn ?? ((message) => log2.warn(message));
|
|
771
|
+
const info = options.info ?? ((message) => log2.info(message));
|
|
772
|
+
const error = options.error ?? ((message) => log2.error(message));
|
|
773
|
+
options.state.ready = false;
|
|
774
|
+
options.state.lastError = null;
|
|
775
|
+
options.state.degraded = false;
|
|
776
|
+
if (options.override) {
|
|
777
|
+
options.openGate();
|
|
778
|
+
options.state.ready = true;
|
|
779
|
+
error(
|
|
780
|
+
"CRITICAL: emergency readiness override enabled; exposing a cold search backend to traffic"
|
|
781
|
+
);
|
|
782
|
+
return "overridden";
|
|
783
|
+
}
|
|
784
|
+
if (options.skipWarmup?.()) {
|
|
785
|
+
options.openGate();
|
|
786
|
+
options.state.ready = true;
|
|
787
|
+
info("Standalone init gate opened without search warm-up (search intentionally disabled)");
|
|
788
|
+
return "search-disabled";
|
|
789
|
+
}
|
|
790
|
+
let removeDeferredShutdownListener = () => void 0;
|
|
791
|
+
const deferredShutdown = new Promise((resolve) => {
|
|
792
|
+
if (options.shutdownSignal?.aborted) {
|
|
793
|
+
resolve("shutdown");
|
|
794
|
+
return;
|
|
795
|
+
}
|
|
796
|
+
const onDeferredShutdown = () => resolve("shutdown");
|
|
797
|
+
options.shutdownSignal?.addEventListener("abort", onDeferredShutdown, { once: true });
|
|
798
|
+
removeDeferredShutdownListener = () => options.shutdownSignal?.removeEventListener("abort", onDeferredShutdown);
|
|
799
|
+
});
|
|
800
|
+
try {
|
|
801
|
+
const deferredOutcome = await Promise.race([
|
|
802
|
+
options.deferredReady.then(() => "ready"),
|
|
803
|
+
deferredShutdown
|
|
804
|
+
]);
|
|
805
|
+
if (deferredOutcome === "shutdown") return "cancelled";
|
|
806
|
+
} catch (err) {
|
|
807
|
+
if (options.shutdownSignal?.aborted) return "cancelled";
|
|
808
|
+
options.state.lastError = err instanceof Error ? err.name : typeof err;
|
|
809
|
+
warn(`Standalone deferred initialization failed; warm-up retries will continue: ${err}`);
|
|
810
|
+
} finally {
|
|
811
|
+
removeDeferredShutdownListener();
|
|
812
|
+
}
|
|
813
|
+
if (options.shutdownSignal?.aborted) return "cancelled";
|
|
814
|
+
const lifecycleAbort = new AbortController();
|
|
815
|
+
const onShutdown = () => lifecycleAbort.abort(options.shutdownSignal?.reason);
|
|
816
|
+
options.shutdownSignal?.addEventListener("abort", onShutdown, { once: true });
|
|
817
|
+
try {
|
|
818
|
+
while (!lifecycleAbort.signal.aborted) {
|
|
819
|
+
if (options.skipWarmup?.()) {
|
|
820
|
+
options.state.degraded = false;
|
|
821
|
+
options.openGate();
|
|
822
|
+
options.state.ready = true;
|
|
823
|
+
info("Standalone init gate opened without search warm-up (search intentionally disabled)");
|
|
824
|
+
return "search-disabled";
|
|
825
|
+
}
|
|
826
|
+
options.state.warmupAttempts += 1;
|
|
827
|
+
const warmupAbort = new AbortController();
|
|
828
|
+
const onLifecycleAbort = () => warmupAbort.abort(lifecycleAbort.signal.reason);
|
|
829
|
+
lifecycleAbort.signal.addEventListener("abort", onLifecycleAbort, { once: true });
|
|
830
|
+
const timeout = Promise.withResolvers();
|
|
831
|
+
let timedOut = false;
|
|
832
|
+
const timer = setTimeout(() => {
|
|
833
|
+
timedOut = true;
|
|
834
|
+
warmupAbort.abort();
|
|
835
|
+
timeout.reject(new Error(`startup warm-up timed out after ${timeoutMs}ms`));
|
|
836
|
+
}, timeoutMs);
|
|
837
|
+
timer.unref();
|
|
838
|
+
try {
|
|
839
|
+
const attempt = async () => {
|
|
840
|
+
if (options.prepareWarmup && !await options.prepareWarmup(warmupAbort.signal)) {
|
|
841
|
+
throw new StartupSyncPendingError();
|
|
842
|
+
}
|
|
843
|
+
return options.warmup(warmupAbort.signal);
|
|
844
|
+
};
|
|
845
|
+
await Promise.race([attempt(), timeout.promise]);
|
|
846
|
+
if (lifecycleAbort.signal.aborted) return "cancelled";
|
|
847
|
+
const recovered = options.state.degraded === true;
|
|
848
|
+
options.state.lastError = null;
|
|
849
|
+
options.state.degraded = false;
|
|
850
|
+
options.openGate();
|
|
851
|
+
options.state.ready = true;
|
|
852
|
+
info(
|
|
853
|
+
`Standalone init gate opened after search warm-up attempt ${options.state.warmupAttempts}${recovered ? " (recovered from degraded mode)" : ""}`
|
|
854
|
+
);
|
|
855
|
+
return "warmed";
|
|
856
|
+
} catch (err) {
|
|
857
|
+
if (lifecycleAbort.signal.aborted) return "cancelled";
|
|
858
|
+
options.state.lastError = timedOut ? "TimeoutError" : err instanceof Error ? err.name : typeof err;
|
|
859
|
+
warn(
|
|
860
|
+
timedOut ? `Standalone startup warm-up attempt ${options.state.warmupAttempts} timed out after ${timeoutMs}ms; retrying in ${retryIntervalMs}ms` : `Standalone startup warm-up attempt ${options.state.warmupAttempts} failed (${options.state.lastError}); retrying in ${retryIntervalMs}ms`
|
|
861
|
+
);
|
|
862
|
+
if (degradedAfterAttempts > 0 && !options.state.ready && options.state.warmupAttempts >= degradedAfterAttempts) {
|
|
863
|
+
options.state.degraded = true;
|
|
864
|
+
options.openGate();
|
|
865
|
+
options.state.ready = true;
|
|
866
|
+
warn(
|
|
867
|
+
`Standalone init gate opened in DEGRADED mode after ${options.state.warmupAttempts} failed search warm-up attempts (${options.state.lastError}); recall keeps serving via fallback retrieval and warm-up retries continue in the background`
|
|
868
|
+
);
|
|
869
|
+
}
|
|
870
|
+
} finally {
|
|
871
|
+
clearTimeout(timer);
|
|
872
|
+
lifecycleAbort.signal.removeEventListener("abort", onLifecycleAbort);
|
|
873
|
+
}
|
|
874
|
+
await abortableDelay(retryIntervalMs, lifecycleAbort.signal);
|
|
875
|
+
}
|
|
876
|
+
return "cancelled";
|
|
877
|
+
} finally {
|
|
878
|
+
options.shutdownSignal?.removeEventListener("abort", onShutdown);
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
|
|
722
882
|
// src/index.ts
|
|
723
883
|
function parseServerPort(value, source) {
|
|
724
884
|
const port = typeof value === "string" ? Number(value.trim()) : value;
|
|
@@ -760,6 +920,14 @@ function parseOptionalBoolean(value, source) {
|
|
|
760
920
|
}
|
|
761
921
|
throw new Error(`Invalid ${source}: expected a boolean`);
|
|
762
922
|
}
|
|
923
|
+
function parseOptionalNonNegativeInteger(value, source) {
|
|
924
|
+
if (value === void 0) return void 0;
|
|
925
|
+
const parsed = typeof value === "string" ? value.trim() === "" ? Number.NaN : Number(value.trim()) : value;
|
|
926
|
+
if (typeof parsed !== "number" || !Number.isInteger(parsed) || parsed < 0) {
|
|
927
|
+
throw new Error(`Invalid ${source}: expected a non-negative integer`);
|
|
928
|
+
}
|
|
929
|
+
return parsed;
|
|
930
|
+
}
|
|
763
931
|
function parseServerConfig(raw, options) {
|
|
764
932
|
return {
|
|
765
933
|
host: parseOptionalNonEmptyString(raw.host, "server.host") ?? "127.0.0.1",
|
|
@@ -778,7 +946,11 @@ function parseServerConfig(raw, options) {
|
|
|
778
946
|
adminConsoleEnabled: parseOptionalBoolean(raw.adminConsoleEnabled, "server.adminConsoleEnabled") ?? false,
|
|
779
947
|
adminConsolePublicDir: parseOptionalString(raw.adminConsolePublicDir, "server.adminConsolePublicDir"),
|
|
780
948
|
adminConsolePrefillToken: parseOptionalBoolean(raw.adminConsolePrefillToken, "server.adminConsolePrefillToken") ?? false,
|
|
781
|
-
readinessOverride: parseOptionalBoolean(raw.readinessOverride, "server.readinessOverride") ?? false
|
|
949
|
+
readinessOverride: parseOptionalBoolean(raw.readinessOverride, "server.readinessOverride") ?? false,
|
|
950
|
+
readinessDegradedAfterAttempts: parseOptionalNonNegativeInteger(
|
|
951
|
+
raw.readinessDegradedAfterAttempts,
|
|
952
|
+
"server.readinessDegradedAfterAttempts"
|
|
953
|
+
) ?? STARTUP_DEGRADED_AFTER_ATTEMPTS
|
|
782
954
|
};
|
|
783
955
|
}
|
|
784
956
|
function resolveUserPath(value) {
|
|
@@ -1229,154 +1401,16 @@ function createAdminControls(configPath, config, serverConfig) {
|
|
|
1229
1401
|
}
|
|
1230
1402
|
};
|
|
1231
1403
|
}
|
|
1232
|
-
function abortableDelay(ms, signal) {
|
|
1233
|
-
if (signal.aborted) return Promise.resolve();
|
|
1234
|
-
const { promise, resolve } = Promise.withResolvers();
|
|
1235
|
-
const timer = setTimeout(resolve, ms);
|
|
1236
|
-
const onAbort = () => {
|
|
1237
|
-
clearTimeout(timer);
|
|
1238
|
-
resolve();
|
|
1239
|
-
};
|
|
1240
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
1241
|
-
return promise.finally(() => signal.removeEventListener("abort", onAbort));
|
|
1242
|
-
}
|
|
1243
|
-
var STARTUP_WARMUP_TIMEOUT_MS = 2e4;
|
|
1244
|
-
var STARTUP_WARMUP_RETRY_INTERVAL_MS = 3e4;
|
|
1245
|
-
var StartupWarmupDegradationError = class extends Error {
|
|
1246
|
-
constructor(code) {
|
|
1247
|
-
super(`startup search degraded: ${code}`);
|
|
1248
|
-
this.name = "StartupWarmupDegradationError";
|
|
1249
|
-
}
|
|
1250
|
-
};
|
|
1251
|
-
var StartupSyncPendingError = class extends Error {
|
|
1252
|
-
constructor() {
|
|
1253
|
-
super("startup search sync is not complete");
|
|
1254
|
-
this.name = "StartupSyncPendingError";
|
|
1255
|
-
}
|
|
1256
|
-
};
|
|
1257
|
-
async function runStartupSearchWarmup(options) {
|
|
1258
|
-
let degradationCode;
|
|
1259
|
-
await options.search((code) => {
|
|
1260
|
-
degradationCode = code;
|
|
1261
|
-
});
|
|
1262
|
-
if (options.signal.aborted) return;
|
|
1263
|
-
if (degradationCode) throw new StartupWarmupDegradationError(degradationCode);
|
|
1264
|
-
if (!options.isAvailable()) {
|
|
1265
|
-
throw new StartupWarmupDegradationError("backend_unavailable");
|
|
1266
|
-
}
|
|
1267
|
-
}
|
|
1268
|
-
async function completeStartupReadiness(options) {
|
|
1269
|
-
const timeoutMs = options.timeoutMs ?? STARTUP_WARMUP_TIMEOUT_MS;
|
|
1270
|
-
const retryIntervalMs = options.retryIntervalMs ?? STARTUP_WARMUP_RETRY_INTERVAL_MS;
|
|
1271
|
-
const warn = options.warn ?? ((message) => log2.warn(message));
|
|
1272
|
-
const info = options.info ?? ((message) => log2.info(message));
|
|
1273
|
-
const error = options.error ?? ((message) => log2.error(message));
|
|
1274
|
-
options.state.ready = false;
|
|
1275
|
-
options.state.lastError = null;
|
|
1276
|
-
if (options.override) {
|
|
1277
|
-
options.openGate();
|
|
1278
|
-
options.state.ready = true;
|
|
1279
|
-
error(
|
|
1280
|
-
"CRITICAL: emergency readiness override enabled; exposing a cold search backend to traffic"
|
|
1281
|
-
);
|
|
1282
|
-
return "overridden";
|
|
1283
|
-
}
|
|
1284
|
-
if (options.skipWarmup?.()) {
|
|
1285
|
-
options.openGate();
|
|
1286
|
-
options.state.ready = true;
|
|
1287
|
-
info("Standalone init gate opened without search warm-up (search intentionally disabled)");
|
|
1288
|
-
return "search-disabled";
|
|
1289
|
-
}
|
|
1290
|
-
let removeDeferredShutdownListener = () => void 0;
|
|
1291
|
-
const deferredShutdown = new Promise((resolve) => {
|
|
1292
|
-
if (options.shutdownSignal?.aborted) {
|
|
1293
|
-
resolve("shutdown");
|
|
1294
|
-
return;
|
|
1295
|
-
}
|
|
1296
|
-
const onDeferredShutdown = () => resolve("shutdown");
|
|
1297
|
-
options.shutdownSignal?.addEventListener("abort", onDeferredShutdown, { once: true });
|
|
1298
|
-
removeDeferredShutdownListener = () => options.shutdownSignal?.removeEventListener("abort", onDeferredShutdown);
|
|
1299
|
-
});
|
|
1300
|
-
try {
|
|
1301
|
-
const deferredOutcome = await Promise.race([
|
|
1302
|
-
options.deferredReady.then(() => "ready"),
|
|
1303
|
-
deferredShutdown
|
|
1304
|
-
]);
|
|
1305
|
-
if (deferredOutcome === "shutdown") return "cancelled";
|
|
1306
|
-
} catch (err) {
|
|
1307
|
-
if (options.shutdownSignal?.aborted) return "cancelled";
|
|
1308
|
-
options.state.lastError = err instanceof Error ? err.name : typeof err;
|
|
1309
|
-
warn(`Standalone deferred initialization failed; warm-up retries will continue: ${err}`);
|
|
1310
|
-
} finally {
|
|
1311
|
-
removeDeferredShutdownListener();
|
|
1312
|
-
}
|
|
1313
|
-
if (options.shutdownSignal?.aborted) return "cancelled";
|
|
1314
|
-
const lifecycleAbort = new AbortController();
|
|
1315
|
-
const onShutdown = () => lifecycleAbort.abort(options.shutdownSignal?.reason);
|
|
1316
|
-
options.shutdownSignal?.addEventListener("abort", onShutdown, { once: true });
|
|
1317
|
-
try {
|
|
1318
|
-
while (!lifecycleAbort.signal.aborted) {
|
|
1319
|
-
if (options.skipWarmup?.()) {
|
|
1320
|
-
options.openGate();
|
|
1321
|
-
options.state.ready = true;
|
|
1322
|
-
info("Standalone init gate opened without search warm-up (search intentionally disabled)");
|
|
1323
|
-
return "search-disabled";
|
|
1324
|
-
}
|
|
1325
|
-
options.state.warmupAttempts += 1;
|
|
1326
|
-
const warmupAbort = new AbortController();
|
|
1327
|
-
const onLifecycleAbort = () => warmupAbort.abort(lifecycleAbort.signal.reason);
|
|
1328
|
-
lifecycleAbort.signal.addEventListener("abort", onLifecycleAbort, { once: true });
|
|
1329
|
-
const timeout = Promise.withResolvers();
|
|
1330
|
-
let timedOut = false;
|
|
1331
|
-
const timer = setTimeout(() => {
|
|
1332
|
-
timedOut = true;
|
|
1333
|
-
warmupAbort.abort();
|
|
1334
|
-
timeout.reject(new Error(`startup warm-up timed out after ${timeoutMs}ms`));
|
|
1335
|
-
}, timeoutMs);
|
|
1336
|
-
timer.unref();
|
|
1337
|
-
try {
|
|
1338
|
-
const attempt = async () => {
|
|
1339
|
-
if (options.prepareWarmup && !await options.prepareWarmup(warmupAbort.signal)) {
|
|
1340
|
-
throw new StartupSyncPendingError();
|
|
1341
|
-
}
|
|
1342
|
-
return options.warmup(warmupAbort.signal);
|
|
1343
|
-
};
|
|
1344
|
-
await Promise.race([attempt(), timeout.promise]);
|
|
1345
|
-
if (lifecycleAbort.signal.aborted) return "cancelled";
|
|
1346
|
-
options.state.lastError = null;
|
|
1347
|
-
options.openGate();
|
|
1348
|
-
options.state.ready = true;
|
|
1349
|
-
info(
|
|
1350
|
-
`Standalone init gate opened after search warm-up attempt ${options.state.warmupAttempts}`
|
|
1351
|
-
);
|
|
1352
|
-
return "warmed";
|
|
1353
|
-
} catch (err) {
|
|
1354
|
-
if (lifecycleAbort.signal.aborted) return "cancelled";
|
|
1355
|
-
options.state.lastError = timedOut ? "TimeoutError" : err instanceof Error ? err.name : typeof err;
|
|
1356
|
-
warn(
|
|
1357
|
-
timedOut ? `Standalone startup warm-up attempt ${options.state.warmupAttempts} timed out after ${timeoutMs}ms; retrying in ${retryIntervalMs}ms` : `Standalone startup warm-up attempt ${options.state.warmupAttempts} failed (${options.state.lastError}); retrying in ${retryIntervalMs}ms`
|
|
1358
|
-
);
|
|
1359
|
-
} finally {
|
|
1360
|
-
clearTimeout(timer);
|
|
1361
|
-
lifecycleAbort.signal.removeEventListener("abort", onLifecycleAbort);
|
|
1362
|
-
}
|
|
1363
|
-
await abortableDelay(retryIntervalMs, lifecycleAbort.signal);
|
|
1364
|
-
}
|
|
1365
|
-
return "cancelled";
|
|
1366
|
-
} finally {
|
|
1367
|
-
options.shutdownSignal?.removeEventListener("abort", onShutdown);
|
|
1368
|
-
}
|
|
1369
|
-
}
|
|
1370
1404
|
async function cleanupFailedStartup(orchestrator, httpServer) {
|
|
1371
1405
|
try {
|
|
1372
1406
|
await httpServer.stop();
|
|
1373
1407
|
} catch (err) {
|
|
1374
|
-
|
|
1408
|
+
log3.warn(`HTTP startup failure cleanup could not stop server: ${err}`);
|
|
1375
1409
|
}
|
|
1376
1410
|
try {
|
|
1377
1411
|
await orchestrator.destroy();
|
|
1378
1412
|
} catch (err) {
|
|
1379
|
-
|
|
1413
|
+
log3.warn(`HTTP startup failure cleanup could not destroy orchestrator: ${err}`);
|
|
1380
1414
|
}
|
|
1381
1415
|
}
|
|
1382
1416
|
async function startServer(options) {
|
|
@@ -1385,7 +1419,7 @@ async function startServer(options) {
|
|
|
1385
1419
|
if (!driverProbe.ok) {
|
|
1386
1420
|
const detailSuffix = driverProbe.detail ? ` (${driverProbe.detail})` : "";
|
|
1387
1421
|
const abiSuffix = driverProbe.nativeBindingMismatch ? " \u2014 the binding was built for a different Node.js ABI; rebuild it (`node scripts/ensure-better-sqlite3.mjs` or `pnpm rebuild better-sqlite3`)" : "";
|
|
1388
|
-
|
|
1422
|
+
log3.error(
|
|
1389
1423
|
`better-sqlite3 native driver failed to load under the running process${detailSuffix}${abiSuffix}. SQLite-backed features (memory projection) will fall back to slower full-corpus scans until fixed.`
|
|
1390
1424
|
);
|
|
1391
1425
|
}
|
|
@@ -1407,14 +1441,14 @@ async function startServer(options) {
|
|
|
1407
1441
|
const parsedServerConfig = parseServerConfig(serverConfig, { portSource });
|
|
1408
1442
|
const config = parseConfig(remnicConfig);
|
|
1409
1443
|
initLogger(void 0, config.debug);
|
|
1410
|
-
|
|
1444
|
+
log3.debug(`debug logging enabled from config (${resolvedConfigPath.source})`);
|
|
1411
1445
|
const orchestrator = new Orchestrator(config);
|
|
1412
1446
|
await orchestrator.initialize();
|
|
1413
1447
|
const service = new EngramAccessService(orchestrator);
|
|
1414
|
-
const readiness = { ready: false, warmupAttempts: 0, lastError: null };
|
|
1448
|
+
const readiness = { ready: false, warmupAttempts: 0, lastError: null, degraded: false };
|
|
1415
1449
|
const authToken = parsedServerConfig.authToken ?? readCompatEnv("REMNIC_AUTH_TOKEN", "ENGRAM_AUTH_TOKEN") ?? "";
|
|
1416
1450
|
if (!authToken && getAllValidTokens().length === 0) {
|
|
1417
|
-
|
|
1451
|
+
log3.warn("No auth token set \u2014 server will reject all requests. Set REMNIC_AUTH_TOKEN, server.authToken in config, or generate tokens with 'remnic token generate'.");
|
|
1418
1452
|
}
|
|
1419
1453
|
const oauthConfig = applyOAuthEnvOverrides(serverConfig.oauth);
|
|
1420
1454
|
const oauthRequestHandler = buildOAuthRequestHandler(oauthConfig);
|
|
@@ -1494,6 +1528,7 @@ async function startServer(options) {
|
|
|
1494
1528
|
prepareWarmup: ensureStartupSync,
|
|
1495
1529
|
state: readiness,
|
|
1496
1530
|
override: parsedServerConfig.readinessOverride,
|
|
1531
|
+
degradedAfterAttempts: parsedServerConfig.readinessDegradedAfterAttempts,
|
|
1497
1532
|
skipWarmup: () => orchestrator.qmd.debugStatus() === "backend=noop",
|
|
1498
1533
|
openGate: () => {
|
|
1499
1534
|
readiness.ready = true;
|
|
@@ -1523,47 +1558,47 @@ async function startServer(options) {
|
|
|
1523
1558
|
httpServer.stop = stop;
|
|
1524
1559
|
orchestrator.deferredReady.then(() => {
|
|
1525
1560
|
if (startupSyncAbort.signal.aborted) {
|
|
1526
|
-
|
|
1561
|
+
log3.debug("QMD startup-sync: cancelled before deferred init completed");
|
|
1527
1562
|
return;
|
|
1528
1563
|
}
|
|
1529
1564
|
if (!config.qmdEnabled || orchestrator.qmd.debugStatus() === "backend=noop") {
|
|
1530
|
-
|
|
1565
|
+
log3.debug("QMD startup-sync: search disabled or noop backend, skipping retries");
|
|
1531
1566
|
return;
|
|
1532
1567
|
}
|
|
1533
1568
|
const needsRetry = !orchestrator.qmd.isAvailable() || !orchestrator.deferredSyncSucceeded;
|
|
1534
1569
|
if (!needsRetry) {
|
|
1535
|
-
|
|
1570
|
+
log3.debug("QMD startup-sync: deferred init completed successfully, no retries needed");
|
|
1536
1571
|
return;
|
|
1537
1572
|
}
|
|
1538
1573
|
const RETRY_DELAYS_MS = [5e3, 15e3, 3e4, 6e4, 12e4];
|
|
1539
1574
|
if (startupSyncAbort.signal.aborted) {
|
|
1540
|
-
|
|
1575
|
+
log3.debug("QMD startup-sync retry: cancelled before retry task started");
|
|
1541
1576
|
return;
|
|
1542
1577
|
}
|
|
1543
1578
|
(async () => {
|
|
1544
1579
|
for (const delay of RETRY_DELAYS_MS) {
|
|
1545
1580
|
await abortableDelay(delay, startupSyncAbort.signal);
|
|
1546
1581
|
if (startupSyncAbort.signal.aborted) {
|
|
1547
|
-
|
|
1582
|
+
log3.debug("QMD startup-sync retry: cancelled by shutdown");
|
|
1548
1583
|
return;
|
|
1549
1584
|
}
|
|
1550
1585
|
const synced = await ensureStartupSync(startupSyncAbort.signal);
|
|
1551
1586
|
if (!synced) {
|
|
1552
1587
|
if (orchestrator.qmd.debugStatus() === "backend=noop") {
|
|
1553
|
-
|
|
1588
|
+
log3.debug("QMD startup-sync retry: search intentionally disabled; stopping retries");
|
|
1554
1589
|
return;
|
|
1555
1590
|
}
|
|
1556
|
-
|
|
1591
|
+
log3.debug(`QMD startup-sync retry: not available yet (next retry in ${RETRY_DELAYS_MS[RETRY_DELAYS_MS.indexOf(delay) + 1] ?? "n/a"}ms)`);
|
|
1557
1592
|
continue;
|
|
1558
1593
|
}
|
|
1559
1594
|
return;
|
|
1560
1595
|
}
|
|
1561
|
-
|
|
1596
|
+
log3.warn("QMD startup-sync retry: exhausted all retries; search index may be stale");
|
|
1562
1597
|
})().catch((err) => {
|
|
1563
|
-
|
|
1598
|
+
log3.warn(`QMD startup-sync retry: unexpected error: ${err}`);
|
|
1564
1599
|
});
|
|
1565
1600
|
}).catch((err) => {
|
|
1566
|
-
|
|
1601
|
+
log3.warn(`Deferred init error: ${err}`);
|
|
1567
1602
|
});
|
|
1568
1603
|
return { config, service, httpServer, host, port, stop, cancelStartupSync: () => startupSyncAbort.abort(), abortDeferredInit: () => orchestrator.abortDeferredInit() };
|
|
1569
1604
|
}
|