@openpalm/lib 0.10.2 → 0.11.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +7 -3
- package/src/control-plane/admin-token.ts +73 -0
- package/src/control-plane/akm-vault.test.ts +105 -0
- package/src/control-plane/akm-vault.ts +307 -0
- package/src/control-plane/channels.ts +3 -3
- package/src/control-plane/cleanup-guardrails.test.ts +8 -9
- package/src/control-plane/compose-args.test.ts +25 -24
- package/src/control-plane/compose-errors.test.ts +106 -0
- package/src/control-plane/compose-errors.ts +117 -0
- package/src/control-plane/config-persistence.ts +103 -65
- package/src/control-plane/core-assets.test.ts +104 -0
- package/src/control-plane/core-assets.ts +54 -57
- package/src/control-plane/docker.ts +55 -21
- package/src/control-plane/env.test.ts +25 -1
- package/src/control-plane/env.ts +80 -0
- package/src/control-plane/home.ts +66 -69
- package/src/control-plane/host-opencode.test.ts +260 -0
- package/src/control-plane/host-opencode.ts +229 -0
- package/src/control-plane/install-edge-cases.test.ts +187 -289
- package/src/control-plane/install-lock.ts +157 -0
- package/src/control-plane/lifecycle.ts +34 -65
- package/src/control-plane/markdown-task.ts +200 -0
- package/src/control-plane/migrate-0110.test.ts +177 -0
- package/src/control-plane/migrate-0110.ts +99 -0
- package/src/control-plane/paths.ts +82 -0
- package/src/control-plane/provider-config.ts +2 -2
- package/src/control-plane/provider-models.ts +154 -0
- package/src/control-plane/registry-components.test.ts +105 -27
- package/src/control-plane/registry.test.ts +49 -47
- package/src/control-plane/registry.ts +71 -50
- package/src/control-plane/rollback.ts +17 -16
- package/src/control-plane/scheduler.ts +75 -262
- package/src/control-plane/secret-backend.test.ts +98 -111
- package/src/control-plane/secret-backend.ts +221 -181
- package/src/control-plane/secret-mappings.ts +4 -8
- package/src/control-plane/secrets.ts +93 -51
- package/src/control-plane/setup-config.schema.json +5 -17
- package/src/control-plane/setup-status.ts +9 -29
- package/src/control-plane/setup-validation.ts +23 -23
- package/src/control-plane/setup.test.ts +138 -239
- package/src/control-plane/setup.ts +215 -130
- package/src/control-plane/skeleton-guardrail.test.ts +151 -0
- package/src/control-plane/spec-to-env.test.ts +59 -58
- package/src/control-plane/spec-to-env.ts +52 -142
- package/src/control-plane/spec-validator.ts +2 -99
- package/src/control-plane/stack-spec.test.ts +21 -77
- package/src/control-plane/stack-spec.ts +7 -83
- package/src/control-plane/types.ts +12 -28
- package/src/control-plane/ui-assets.ts +349 -0
- package/src/control-plane/validate.ts +44 -79
- package/src/index.ts +86 -48
- package/src/logger.test.ts +228 -0
- package/src/logger.ts +71 -1
- package/src/provider-constants.ts +22 -1
- package/src/control-plane/audit.ts +0 -40
- package/src/control-plane/env-schema-validation.test.ts +0 -118
- package/src/control-plane/memory-config.ts +0 -298
- package/src/control-plane/redact-schema.ts +0 -50
package/src/logger.ts
CHANGED
|
@@ -1,8 +1,78 @@
|
|
|
1
1
|
export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* In-house redactor. Returns `'***REDACTED***'` when `key` names something
|
|
5
|
+
* that looks like a secret (token, key, secret, password, hmac). Replaces
|
|
6
|
+
* the value-masking that varlock used to do for log output.
|
|
7
|
+
*
|
|
8
|
+
* The pattern matches the bare word at the start or end of the key, using
|
|
9
|
+
* underscore as a word boundary. This avoids substring false positives
|
|
10
|
+
* like `MONKEY` (contains `_KEY`? no, but the un-anchored pattern used
|
|
11
|
+
* to match the substring `KEY` even without an underscore) and
|
|
12
|
+
* `PACKET_SIZE` (does not actually contain `_KEY`, but the regex engine
|
|
13
|
+
* with un-anchored alternations was sloppy enough to invite future bugs).
|
|
14
|
+
*
|
|
15
|
+
* Examples:
|
|
16
|
+
* OP_UI_LOGIN_PASSWORD → sensitive (suffix _PASSWORD)
|
|
17
|
+
* CHANNEL_API_KEY → sensitive (suffix _KEY)
|
|
18
|
+
* CHANNEL_FOO_HMAC → sensitive (suffix _HMAC)
|
|
19
|
+
* HMAC_KEY → sensitive (prefix HMAC_, suffix _KEY)
|
|
20
|
+
* TOKEN → sensitive (bare word)
|
|
21
|
+
* MONKEY → NOT sensitive
|
|
22
|
+
* PACKET_SIZE → NOT sensitive
|
|
23
|
+
*
|
|
24
|
+
* The same predicate is exported as {@link isSensitiveEnvKey} so callers
|
|
25
|
+
* that need to mask only part of a larger payload can short-circuit.
|
|
26
|
+
*/
|
|
27
|
+
const REDACT_PATTERN = /(?:^|_)(?:TOKEN|SECRET|KEY|PASSWORD|HMAC)(?:_|$)/i;
|
|
28
|
+
|
|
29
|
+
export function isSensitiveEnvKey(key: string): boolean {
|
|
30
|
+
return REDACT_PATTERN.test(key);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function redactValue(key: string, value: string): string {
|
|
34
|
+
return isSensitiveEnvKey(key) ? '***REDACTED***' : value;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Recursively walk a structured `extra` payload and mask every value whose
|
|
39
|
+
* own key (or the nearest enclosing object key) matches the sensitivity
|
|
40
|
+
* pattern. The original object is not mutated. Sensitive values of any
|
|
41
|
+
* primitive type (string, number, boolean) are replaced wholesale; nested
|
|
42
|
+
* objects under a sensitive key are still walked so that callers can mix
|
|
43
|
+
* structured payloads with redacted leaves.
|
|
44
|
+
*/
|
|
45
|
+
export function redactExtra<T>(extra: T): T {
|
|
46
|
+
if (extra == null || typeof extra !== 'object') return extra;
|
|
47
|
+
if (Array.isArray(extra)) {
|
|
48
|
+
return extra.map((v) => (v && typeof v === 'object' ? redactExtra(v) : v)) as unknown as T;
|
|
49
|
+
}
|
|
50
|
+
const out: Record<string, unknown> = {};
|
|
51
|
+
for (const [k, v] of Object.entries(extra as Record<string, unknown>)) {
|
|
52
|
+
if (isSensitiveEnvKey(k)) {
|
|
53
|
+
// Redact any non-null primitive (string/number/boolean) under a
|
|
54
|
+
// sensitive key. Nested objects keep being walked so a structured
|
|
55
|
+
// payload like { credentials: { ... } } still gets per-field masking.
|
|
56
|
+
if (v && typeof v === 'object') {
|
|
57
|
+
out[k] = redactExtra(v);
|
|
58
|
+
} else if (v == null) {
|
|
59
|
+
out[k] = v;
|
|
60
|
+
} else {
|
|
61
|
+
out[k] = '***REDACTED***';
|
|
62
|
+
}
|
|
63
|
+
} else if (v && typeof v === 'object') {
|
|
64
|
+
out[k] = redactExtra(v);
|
|
65
|
+
} else {
|
|
66
|
+
out[k] = v;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return out as T;
|
|
70
|
+
}
|
|
71
|
+
|
|
3
72
|
export function createLogger(service: string) {
|
|
4
73
|
function log(level: LogLevel, msg: string, extra?: Record<string, unknown>): void {
|
|
5
|
-
const
|
|
74
|
+
const safeExtra = extra ? redactExtra(extra) : undefined;
|
|
75
|
+
const entry = { ts: new Date().toISOString(), level, service, msg, ...(safeExtra ? { extra: safeExtra } : {}) };
|
|
6
76
|
(level === 'error' || level === 'warn' ? console.error : console.log)(JSON.stringify(entry));
|
|
7
77
|
}
|
|
8
78
|
return {
|
|
@@ -40,19 +40,40 @@ export const PROVIDER_KEY_MAP: Record<string, string> = {
|
|
|
40
40
|
huggingface: "HF_TOKEN",
|
|
41
41
|
};
|
|
42
42
|
|
|
43
|
-
/** Known embedding model dimensions
|
|
43
|
+
/** Known embedding model dimensions. Keyed by `provider/model`. */
|
|
44
44
|
export const EMBEDDING_DIMS: Record<string, number> = {
|
|
45
45
|
"openai/text-embedding-3-small": 1536,
|
|
46
46
|
"openai/text-embedding-3-large": 3072,
|
|
47
47
|
"openai/text-embedding-ada-002": 1536,
|
|
48
48
|
"ollama/nomic-embed-text": 768,
|
|
49
49
|
"ollama/mxbai-embed-large": 1024,
|
|
50
|
+
"ollama/mxbai-embed-large-v1": 1024,
|
|
50
51
|
"ollama/all-minilm": 384,
|
|
51
52
|
"ollama/snowflake-arctic-embed": 1024,
|
|
53
|
+
"model-runner/ai/mxbai-embed-large-v1": 1024,
|
|
54
|
+
"mistral/mistral-embed": 1024,
|
|
52
55
|
"google/text-embedding-004": 768,
|
|
53
56
|
"huggingface/sentence-transformers/all-MiniLM-L6-v2": 384,
|
|
57
|
+
"huggingface/intfloat/multilingual-e5-large": 1024,
|
|
54
58
|
};
|
|
55
59
|
|
|
60
|
+
/**
|
|
61
|
+
* Look up embedding model dimensions. Tries the full key first, then strips
|
|
62
|
+
* any trailing `:tag` from the model name (Ollama-style versions).
|
|
63
|
+
* Returns 0 when no match is found.
|
|
64
|
+
*/
|
|
65
|
+
export function lookupEmbeddingDims(provider: string, model: string): number {
|
|
66
|
+
if (!provider || !model) return 0;
|
|
67
|
+
const key = `${provider}/${model}`;
|
|
68
|
+
if (EMBEDDING_DIMS[key]) return EMBEDDING_DIMS[key];
|
|
69
|
+
const colon = model.lastIndexOf(":");
|
|
70
|
+
if (colon > 0) {
|
|
71
|
+
const bare = `${provider}/${model.slice(0, colon)}`;
|
|
72
|
+
if (EMBEDDING_DIMS[bare]) return EMBEDDING_DIMS[bare];
|
|
73
|
+
}
|
|
74
|
+
return 0;
|
|
75
|
+
}
|
|
76
|
+
|
|
56
77
|
/** Provider display labels for UI. */
|
|
57
78
|
export const PROVIDER_LABELS: Record<string, string> = {
|
|
58
79
|
openai: "OpenAI",
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Audit logging for the OpenPalm control plane.
|
|
3
|
-
*/
|
|
4
|
-
import { mkdirSync, appendFileSync } from "node:fs";
|
|
5
|
-
import type { ControlPlaneState, AuditEntry, CallerType } from "./types.js";
|
|
6
|
-
|
|
7
|
-
const MAX_AUDIT_MEMORY = 1000;
|
|
8
|
-
|
|
9
|
-
export function appendAudit(
|
|
10
|
-
state: ControlPlaneState,
|
|
11
|
-
actor: string,
|
|
12
|
-
action: string,
|
|
13
|
-
args: Record<string, unknown>,
|
|
14
|
-
ok: boolean,
|
|
15
|
-
requestId = "",
|
|
16
|
-
callerType: CallerType = "unknown"
|
|
17
|
-
): void {
|
|
18
|
-
const entry: AuditEntry = {
|
|
19
|
-
at: new Date().toISOString(),
|
|
20
|
-
requestId,
|
|
21
|
-
actor,
|
|
22
|
-
callerType,
|
|
23
|
-
action,
|
|
24
|
-
args,
|
|
25
|
-
ok
|
|
26
|
-
};
|
|
27
|
-
state.audit.push(entry);
|
|
28
|
-
if (state.audit.length > MAX_AUDIT_MEMORY) {
|
|
29
|
-
state.audit = state.audit.slice(-MAX_AUDIT_MEMORY);
|
|
30
|
-
}
|
|
31
|
-
try {
|
|
32
|
-
mkdirSync(state.logsDir, { recursive: true });
|
|
33
|
-
appendFileSync(
|
|
34
|
-
`${state.logsDir}/admin-audit.jsonl`,
|
|
35
|
-
JSON.stringify(entry) + "\n"
|
|
36
|
-
);
|
|
37
|
-
} catch {
|
|
38
|
-
// best-effort persistence
|
|
39
|
-
}
|
|
40
|
-
}
|
|
@@ -1,118 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Test that env schema validation uses the correct nested vault paths.
|
|
3
|
-
*/
|
|
4
|
-
import { describe, test, expect, beforeAll, afterAll } from "bun:test";
|
|
5
|
-
import { mkdirSync, writeFileSync, rmSync, existsSync } from "node:fs";
|
|
6
|
-
import { join } from "node:path";
|
|
7
|
-
import { tmpdir } from "node:os";
|
|
8
|
-
import type { ControlPlaneState } from "./types.js";
|
|
9
|
-
|
|
10
|
-
describe("env schema validation paths", () => {
|
|
11
|
-
let tmpDir: string;
|
|
12
|
-
let state: ControlPlaneState;
|
|
13
|
-
|
|
14
|
-
beforeAll(() => {
|
|
15
|
-
tmpDir = join(tmpdir(), `openpalm-schema-test-${Date.now()}`);
|
|
16
|
-
mkdirSync(join(tmpDir, "vault/user"), { recursive: true });
|
|
17
|
-
mkdirSync(join(tmpDir, "vault/stack"), { recursive: true });
|
|
18
|
-
mkdirSync(join(tmpDir, "data"), { recursive: true });
|
|
19
|
-
mkdirSync(join(tmpDir, "logs"), { recursive: true });
|
|
20
|
-
mkdirSync(join(tmpDir, "config"), { recursive: true });
|
|
21
|
-
|
|
22
|
-
state = {
|
|
23
|
-
adminToken: "test-token",
|
|
24
|
-
assistantToken: "test-assistant",
|
|
25
|
-
setupToken: "test-setup",
|
|
26
|
-
homeDir: tmpDir,
|
|
27
|
-
configDir: join(tmpDir, "config"),
|
|
28
|
-
vaultDir: join(tmpDir, "vault"),
|
|
29
|
-
dataDir: join(tmpDir, "data"),
|
|
30
|
-
logsDir: join(tmpDir, "logs"),
|
|
31
|
-
cacheDir: join(tmpDir, "cache"),
|
|
32
|
-
services: {},
|
|
33
|
-
artifacts: { compose: "" },
|
|
34
|
-
artifactMeta: [],
|
|
35
|
-
audit: [],
|
|
36
|
-
};
|
|
37
|
-
});
|
|
38
|
-
|
|
39
|
-
afterAll(() => {
|
|
40
|
-
if (tmpDir && existsSync(tmpDir)) {
|
|
41
|
-
rmSync(tmpDir, { recursive: true, force: true });
|
|
42
|
-
}
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
test("validation succeeds when no schema files exist (skip mode)", async () => {
|
|
46
|
-
const { validateProposedState } = await import("./validate.js");
|
|
47
|
-
const result = await validateProposedState(state);
|
|
48
|
-
// When schema files don't exist, validation is skipped (no errors)
|
|
49
|
-
expect(result.ok).toBe(true);
|
|
50
|
-
expect(result.errors).toEqual([]);
|
|
51
|
-
});
|
|
52
|
-
|
|
53
|
-
test("schema paths match canonical vault layout", () => {
|
|
54
|
-
const expectedUserSchema = join(tmpDir, "vault/user/user.env.schema");
|
|
55
|
-
const expectedStackSchema = join(tmpDir, "vault/stack/stack.env.schema");
|
|
56
|
-
|
|
57
|
-
writeFileSync(expectedUserSchema, "# test schema\n");
|
|
58
|
-
writeFileSync(expectedStackSchema, "# test schema\n");
|
|
59
|
-
|
|
60
|
-
expect(existsSync(expectedUserSchema)).toBe(true);
|
|
61
|
-
expect(existsSync(expectedStackSchema)).toBe(true);
|
|
62
|
-
|
|
63
|
-
// Old flat paths must NOT exist
|
|
64
|
-
expect(existsSync(join(tmpDir, "vault/user.env.schema"))).toBe(false);
|
|
65
|
-
expect(existsSync(join(tmpDir, "vault/system.env.schema"))).toBe(false);
|
|
66
|
-
});
|
|
67
|
-
|
|
68
|
-
test("validate.ts reads from nested paths, not flat paths", async () => {
|
|
69
|
-
// Write schemas at OLD flat paths — should be ignored
|
|
70
|
-
writeFileSync(join(tmpDir, "vault/user.env.schema"), "OPENAI_API_KEY\n");
|
|
71
|
-
writeFileSync(join(tmpDir, "vault/system.env.schema"), "OP_ADMIN_TOKEN\n");
|
|
72
|
-
// Write env files
|
|
73
|
-
writeFileSync(join(tmpDir, "vault/user/user.env"), "# empty\n");
|
|
74
|
-
writeFileSync(join(tmpDir, "vault/stack/stack.env"), "# empty\n");
|
|
75
|
-
// Delete nested schemas to prove flat paths are ignored
|
|
76
|
-
try { rmSync(join(tmpDir, "vault/user/user.env.schema")); } catch { /* may not exist */ }
|
|
77
|
-
try { rmSync(join(tmpDir, "vault/stack/stack.env.schema")); } catch { /* may not exist */ }
|
|
78
|
-
|
|
79
|
-
const { validateProposedState } = await import("./validate.js");
|
|
80
|
-
const result = await validateProposedState(state);
|
|
81
|
-
// Should pass because nested schemas don't exist (skipped), not because flat schemas were read
|
|
82
|
-
expect(result.ok).toBe(true);
|
|
83
|
-
});
|
|
84
|
-
|
|
85
|
-
test("validation reports warnings for missing required schema keys", async () => {
|
|
86
|
-
// Seed a schema that requires OPENAI_API_KEY
|
|
87
|
-
writeFileSync(join(tmpDir, "vault/user/user.env.schema"), "OPENAI_API_KEY=string\nOWNER_NAME=string\n");
|
|
88
|
-
// Seed an env file that is missing those keys
|
|
89
|
-
writeFileSync(join(tmpDir, "vault/user/user.env"), "# empty env\nSOME_OTHER_KEY=value\n");
|
|
90
|
-
|
|
91
|
-
const { validateProposedState } = await import("./validate.js");
|
|
92
|
-
const result = await validateProposedState(state);
|
|
93
|
-
// The validator should report warnings for missing keys (not errors — env validation is advisory)
|
|
94
|
-
expect(result.warnings.length).toBeGreaterThanOrEqual(0);
|
|
95
|
-
});
|
|
96
|
-
|
|
97
|
-
test("validation handles malformed env file gracefully", async () => {
|
|
98
|
-
writeFileSync(join(tmpDir, "vault/user/user.env.schema"), "OPENAI_API_KEY=string\n");
|
|
99
|
-
// Malformed: no = sign, just random text
|
|
100
|
-
writeFileSync(join(tmpDir, "vault/user/user.env"), "this is not a valid env file\n===\n");
|
|
101
|
-
|
|
102
|
-
const { validateProposedState } = await import("./validate.js");
|
|
103
|
-
const result = await validateProposedState(state);
|
|
104
|
-
// Should not throw — graceful handling
|
|
105
|
-
expect(typeof result.ok).toBe("boolean");
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
test("validation handles empty schema file gracefully", async () => {
|
|
109
|
-
writeFileSync(join(tmpDir, "vault/user/user.env.schema"), "");
|
|
110
|
-
writeFileSync(join(tmpDir, "vault/user/user.env"), "OPENAI_API_KEY=sk-test\n");
|
|
111
|
-
|
|
112
|
-
const { validateProposedState } = await import("./validate.js");
|
|
113
|
-
const result = await validateProposedState(state);
|
|
114
|
-
// Empty schema may cause varlock to report an error — that's fine,
|
|
115
|
-
// the important thing is it doesn't throw/crash
|
|
116
|
-
expect(typeof result.ok).toBe("boolean");
|
|
117
|
-
});
|
|
118
|
-
});
|
|
@@ -1,298 +0,0 @@
|
|
|
1
|
-
/** Memory LLM & Embedding configuration management. */
|
|
2
|
-
import { mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from "node:fs";
|
|
3
|
-
import { readStackEnv } from "./secrets.js";
|
|
4
|
-
import { EMBEDDING_DIMS, PROVIDER_DEFAULT_URLS } from "../provider-constants.js";
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
export type MemoryConfig = {
|
|
8
|
-
mem0: {
|
|
9
|
-
llm: { provider: string; config: Record<string, unknown> };
|
|
10
|
-
embedder: { provider: string; config: Record<string, unknown> };
|
|
11
|
-
vector_store: {
|
|
12
|
-
provider: "sqlite-vec" | "qdrant";
|
|
13
|
-
config: {
|
|
14
|
-
collection_name: string;
|
|
15
|
-
db_path?: string;
|
|
16
|
-
path?: string;
|
|
17
|
-
embedding_model_dims: number;
|
|
18
|
-
};
|
|
19
|
-
};
|
|
20
|
-
};
|
|
21
|
-
memory: { custom_instructions: string };
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
export const EMBED_PROVIDERS = [
|
|
26
|
-
"openai", "ollama", "huggingface", "lmstudio"
|
|
27
|
-
] as const;
|
|
28
|
-
|
|
29
|
-
/** Static model list for Anthropic (no listing API available). */
|
|
30
|
-
const ANTHROPIC_MODELS = [
|
|
31
|
-
"claude-opus-4-6",
|
|
32
|
-
"claude-sonnet-4-6",
|
|
33
|
-
"claude-opus-4-20250514",
|
|
34
|
-
"claude-sonnet-4-20250514",
|
|
35
|
-
"claude-haiku-4-5-20251001",
|
|
36
|
-
"claude-3-5-sonnet-20241022",
|
|
37
|
-
"claude-3-5-haiku-20241022",
|
|
38
|
-
];
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
export function resolveApiKey(apiKeyRef: string, configDir: string): string {
|
|
42
|
-
if (!apiKeyRef) return "";
|
|
43
|
-
if (!apiKeyRef.startsWith("env:")) return apiKeyRef;
|
|
44
|
-
|
|
45
|
-
const varName = apiKeyRef.slice(4);
|
|
46
|
-
if (process.env[varName]) return process.env[varName]!;
|
|
47
|
-
|
|
48
|
-
const secrets = readStackEnv(configDir);
|
|
49
|
-
return secrets[varName] ?? "";
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
export type ModelDiscoveryReason =
|
|
54
|
-
| 'none'
|
|
55
|
-
| 'provider_static'
|
|
56
|
-
| 'provider_http'
|
|
57
|
-
| 'missing_base_url'
|
|
58
|
-
| 'timeout'
|
|
59
|
-
| 'network';
|
|
60
|
-
|
|
61
|
-
export type ProviderModelsResult = {
|
|
62
|
-
models: string[];
|
|
63
|
-
status: 'ok' | 'recoverable_error';
|
|
64
|
-
reason: ModelDiscoveryReason;
|
|
65
|
-
error?: string;
|
|
66
|
-
};
|
|
67
|
-
|
|
68
|
-
const HTTP_STATUS_LABELS: Record<number, string> = {
|
|
69
|
-
401: 'Invalid or missing API key',
|
|
70
|
-
403: 'Access denied — check API key permissions',
|
|
71
|
-
404: 'Endpoint not found — verify the base URL',
|
|
72
|
-
429: 'Rate limited — try again shortly',
|
|
73
|
-
500: 'Provider internal error',
|
|
74
|
-
502: 'Provider returned a bad gateway error',
|
|
75
|
-
503: 'Provider is temporarily unavailable',
|
|
76
|
-
};
|
|
77
|
-
|
|
78
|
-
export async function fetchProviderModels(
|
|
79
|
-
provider: string,
|
|
80
|
-
apiKeyRef: string,
|
|
81
|
-
baseUrl: string,
|
|
82
|
-
configDir: string
|
|
83
|
-
): Promise<ProviderModelsResult> {
|
|
84
|
-
try {
|
|
85
|
-
if (provider === "anthropic") {
|
|
86
|
-
return { models: [...ANTHROPIC_MODELS], status: 'ok', reason: 'provider_static' };
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
const resolvedKey = resolveApiKey(apiKeyRef, configDir);
|
|
90
|
-
|
|
91
|
-
if (provider === "ollama") {
|
|
92
|
-
const base = baseUrl?.trim() || PROVIDER_DEFAULT_URLS.ollama;
|
|
93
|
-
const url = `${base.replace(/\/+$/, "")}/api/tags`;
|
|
94
|
-
const res = await fetch(url, { signal: AbortSignal.timeout(5000) });
|
|
95
|
-
if (!res.ok) {
|
|
96
|
-
return {
|
|
97
|
-
models: [],
|
|
98
|
-
status: 'recoverable_error',
|
|
99
|
-
reason: 'provider_http',
|
|
100
|
-
error: `Ollama API returned ${res.status}: ${(HTTP_STATUS_LABELS[res.status] ?? `HTTP ${res.status}`)}`,
|
|
101
|
-
};
|
|
102
|
-
}
|
|
103
|
-
const data = (await res.json()) as { models?: { name: string }[] };
|
|
104
|
-
const models = (data.models ?? []).map((m) => m.name).sort();
|
|
105
|
-
return { models, status: 'ok', reason: 'none' };
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
const base = baseUrl?.trim() || PROVIDER_DEFAULT_URLS[provider] || "";
|
|
109
|
-
if (!base) {
|
|
110
|
-
return {
|
|
111
|
-
models: [],
|
|
112
|
-
status: 'recoverable_error',
|
|
113
|
-
reason: 'missing_base_url',
|
|
114
|
-
error: `No base URL configured for provider "${provider}"`,
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
const url = `${base.replace(/\/+$/, "")}/v1/models`;
|
|
118
|
-
|
|
119
|
-
const headers: Record<string, string> = {};
|
|
120
|
-
if (resolvedKey) {
|
|
121
|
-
headers["Authorization"] = `Bearer ${resolvedKey}`;
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
const res = await fetch(url, { headers, signal: AbortSignal.timeout(5000) });
|
|
125
|
-
if (!res.ok) {
|
|
126
|
-
let detail = '';
|
|
127
|
-
try {
|
|
128
|
-
const json = JSON.parse(await res.text()) as Record<string, unknown>;
|
|
129
|
-
const errObj = json.error as Record<string, unknown> | string | undefined;
|
|
130
|
-
detail = (typeof errObj === 'object' && errObj !== null && typeof errObj.message === 'string') ? errObj.message
|
|
131
|
-
: typeof errObj === 'string' ? errObj
|
|
132
|
-
: typeof json.message === 'string' ? json.message
|
|
133
|
-
: typeof json.detail === 'string' ? json.detail : '';
|
|
134
|
-
} catch { /* ignore parse errors */ }
|
|
135
|
-
return {
|
|
136
|
-
models: [],
|
|
137
|
-
status: 'recoverable_error',
|
|
138
|
-
reason: 'provider_http',
|
|
139
|
-
error: detail
|
|
140
|
-
? `Provider API returned ${res.status}: ${detail}`
|
|
141
|
-
: `Provider API returned ${res.status}: ${(HTTP_STATUS_LABELS[res.status] ?? `HTTP ${res.status}`)}`,
|
|
142
|
-
};
|
|
143
|
-
}
|
|
144
|
-
const data = (await res.json()) as { data?: { id: string }[] };
|
|
145
|
-
const models = (data.data ?? []).map((m) => m.id).sort();
|
|
146
|
-
return { models, status: 'ok', reason: 'none' };
|
|
147
|
-
} catch (err) {
|
|
148
|
-
const message =
|
|
149
|
-
err instanceof Error && err.name === "TimeoutError"
|
|
150
|
-
? "Request timed out after 5s"
|
|
151
|
-
: String(err);
|
|
152
|
-
return {
|
|
153
|
-
models: [],
|
|
154
|
-
status: 'recoverable_error',
|
|
155
|
-
reason: err instanceof Error && err.name === 'TimeoutError' ? 'timeout' : 'network',
|
|
156
|
-
error: message,
|
|
157
|
-
};
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
export function getDefaultConfig(): MemoryConfig {
|
|
163
|
-
return {
|
|
164
|
-
mem0: {
|
|
165
|
-
llm: {
|
|
166
|
-
provider: "openai",
|
|
167
|
-
config: {
|
|
168
|
-
model: "gpt-4o-mini",
|
|
169
|
-
temperature: 0.1,
|
|
170
|
-
max_tokens: 2000,
|
|
171
|
-
api_key: "env:OPENAI_API_KEY",
|
|
172
|
-
},
|
|
173
|
-
},
|
|
174
|
-
embedder: {
|
|
175
|
-
provider: "openai",
|
|
176
|
-
config: {
|
|
177
|
-
model: "text-embedding-3-small",
|
|
178
|
-
api_key: "env:OPENAI_API_KEY",
|
|
179
|
-
},
|
|
180
|
-
},
|
|
181
|
-
vector_store: {
|
|
182
|
-
provider: "sqlite-vec",
|
|
183
|
-
config: {
|
|
184
|
-
collection_name: "memory",
|
|
185
|
-
db_path: "/data/memory.db",
|
|
186
|
-
embedding_model_dims: 1536,
|
|
187
|
-
},
|
|
188
|
-
},
|
|
189
|
-
},
|
|
190
|
-
memory: { custom_instructions: "" },
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
export function readMemoryConfig(dataDir: string): MemoryConfig {
|
|
196
|
-
const path = `${dataDir}/memory/default_config.json`;
|
|
197
|
-
if (!existsSync(path)) return getDefaultConfig();
|
|
198
|
-
try {
|
|
199
|
-
return JSON.parse(readFileSync(path, "utf-8")) as MemoryConfig;
|
|
200
|
-
} catch {
|
|
201
|
-
return getDefaultConfig();
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
export function writeMemoryConfig(dataDir: string, config: MemoryConfig): void {
|
|
206
|
-
mkdirSync(`${dataDir}/memory`, { recursive: true });
|
|
207
|
-
writeFileSync(`${dataDir}/memory/default_config.json`, JSON.stringify(config, null, 2) + "\n");
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
export function ensureMemoryConfig(dataDir: string): void {
|
|
211
|
-
if (existsSync(`${dataDir}/memory/default_config.json`)) return;
|
|
212
|
-
writeMemoryConfig(dataDir, getDefaultConfig());
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
export type VectorDimensionResult = {
|
|
217
|
-
match: boolean;
|
|
218
|
-
currentDims?: number;
|
|
219
|
-
expectedDims: number;
|
|
220
|
-
};
|
|
221
|
-
|
|
222
|
-
export function checkVectorDimensions(
|
|
223
|
-
dataDir: string,
|
|
224
|
-
newConfig: MemoryConfig
|
|
225
|
-
): VectorDimensionResult {
|
|
226
|
-
const expectedDims = newConfig.mem0.vector_store.config.embedding_model_dims;
|
|
227
|
-
const persisted = readMemoryConfig(dataDir);
|
|
228
|
-
const currentDims = persisted.mem0.vector_store.config.embedding_model_dims;
|
|
229
|
-
return { match: currentDims === expectedDims, currentDims, expectedDims };
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
export function resetVectorStore(
|
|
233
|
-
dataDir: string
|
|
234
|
-
): { ok: boolean; error?: string } {
|
|
235
|
-
const persisted = readMemoryConfig(dataDir);
|
|
236
|
-
const configuredPath = persisted.mem0.vector_store.config.db_path;
|
|
237
|
-
|
|
238
|
-
let dbPath: string;
|
|
239
|
-
if (configuredPath && configuredPath.startsWith('/data/')) {
|
|
240
|
-
dbPath = `${dataDir}/memory/${configuredPath.slice('/data/'.length)}`;
|
|
241
|
-
} else if (configuredPath && !configuredPath.startsWith('/')) {
|
|
242
|
-
dbPath = `${dataDir}/memory/${configuredPath}`;
|
|
243
|
-
} else {
|
|
244
|
-
dbPath = `${dataDir}/memory/memory.db`;
|
|
245
|
-
}
|
|
246
|
-
const qdrantPath = `${dataDir}/memory/qdrant`;
|
|
247
|
-
try {
|
|
248
|
-
if (existsSync(dbPath)) {
|
|
249
|
-
rmSync(dbPath, { force: true });
|
|
250
|
-
}
|
|
251
|
-
for (const suffix of ['-wal', '-shm']) {
|
|
252
|
-
const walPath = `${dbPath}${suffix}`;
|
|
253
|
-
if (existsSync(walPath)) rmSync(walPath, { force: true });
|
|
254
|
-
}
|
|
255
|
-
if (existsSync(qdrantPath)) {
|
|
256
|
-
rmSync(qdrantPath, { recursive: true, force: true });
|
|
257
|
-
}
|
|
258
|
-
return { ok: true };
|
|
259
|
-
} catch (err) {
|
|
260
|
-
return { ok: false, error: String(err) };
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
async function callMemoryApi(path: string, init?: RequestInit): Promise<Response> {
|
|
266
|
-
const configured = process.env.MEMORY_API_URL?.trim() || process.env.OP_MEMORY_API_URL?.trim();
|
|
267
|
-
const bases = configured ? [configured.replace(/\/+$/, "")] : ["http://memory:8765", "http://127.0.0.1:8765"];
|
|
268
|
-
const token = process.env.MEMORY_AUTH_TOKEN?.trim();
|
|
269
|
-
const authHeaders: Record<string, string> = token ? { authorization: `Bearer ${token}` } : {};
|
|
270
|
-
let lastError: unknown;
|
|
271
|
-
|
|
272
|
-
for (let i = 0; i < bases.length; i++) {
|
|
273
|
-
try {
|
|
274
|
-
const headers = { ...authHeaders, ...(init?.headers as Record<string, string>) };
|
|
275
|
-
return await fetch(`${bases[i]}${path}`, { ...init, headers });
|
|
276
|
-
} catch (err) {
|
|
277
|
-
lastError = err;
|
|
278
|
-
if (i === bases.length - 1) throw err;
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
throw lastError ?? new Error("Memory API request failed");
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
export async function provisionMemoryUser(
|
|
285
|
-
userId: string,
|
|
286
|
-
): Promise<{ ok: boolean; error?: string }> {
|
|
287
|
-
try {
|
|
288
|
-
const res = await callMemoryApi("/api/v1/users", {
|
|
289
|
-
method: "POST",
|
|
290
|
-
headers: { "content-type": "application/json" },
|
|
291
|
-
body: JSON.stringify({ user_id: userId }),
|
|
292
|
-
signal: AbortSignal.timeout(5_000),
|
|
293
|
-
});
|
|
294
|
-
return { ok: res.ok };
|
|
295
|
-
} catch (err) {
|
|
296
|
-
return { ok: false, error: String(err) };
|
|
297
|
-
}
|
|
298
|
-
}
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Auto-generates a `redact.env.schema` from the canonical secret mappings.
|
|
3
|
-
*
|
|
4
|
-
* This ensures that every env var carrying a secret is marked for redaction
|
|
5
|
-
* by varlock, without requiring manual maintenance of the schema file.
|
|
6
|
-
*/
|
|
7
|
-
import { getCoreSecretMappings } from './secret-mappings.js';
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* Generate a redact.env.schema string from the canonical secret mappings.
|
|
11
|
-
*
|
|
12
|
-
* @param systemEnv - The current system env (used to discover dynamic channel secrets)
|
|
13
|
-
* @returns A complete `@env-spec` schema suitable for varlock redaction
|
|
14
|
-
*/
|
|
15
|
-
export function generateRedactSchema(systemEnv: Record<string, string>): string {
|
|
16
|
-
const lines: string[] = [
|
|
17
|
-
'# OpenPalm — Runtime Redaction Schema (auto-generated)',
|
|
18
|
-
'# Marks env vars as @sensitive so varlock redacts their values from',
|
|
19
|
-
'# stdout/stderr before they reach docker compose logs.',
|
|
20
|
-
'#',
|
|
21
|
-
'# @defaultSensitive=true',
|
|
22
|
-
'# @defaultRequired=false',
|
|
23
|
-
'# ---',
|
|
24
|
-
'',
|
|
25
|
-
];
|
|
26
|
-
|
|
27
|
-
const envKeys = new Set<string>();
|
|
28
|
-
for (const mapping of getCoreSecretMappings(systemEnv)) {
|
|
29
|
-
envKeys.add(mapping.envKey);
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
// Include container-runtime env names that differ from env-file keys
|
|
33
|
-
// (compose maps OP_MEMORY_TOKEN -> MEMORY_AUTH_TOKEN, etc.)
|
|
34
|
-
envKeys.add('ADMIN_TOKEN');
|
|
35
|
-
envKeys.add('MEMORY_AUTH_TOKEN');
|
|
36
|
-
envKeys.add('OPENCODE_SERVER_PASSWORD');
|
|
37
|
-
|
|
38
|
-
// Resolved capability API keys (written to stack.env by spec-to-env)
|
|
39
|
-
envKeys.add('OP_CAP_LLM_API_KEY');
|
|
40
|
-
envKeys.add('OP_CAP_EMBEDDINGS_API_KEY');
|
|
41
|
-
envKeys.add('OP_CAP_TTS_API_KEY');
|
|
42
|
-
envKeys.add('OP_CAP_STT_API_KEY');
|
|
43
|
-
envKeys.add('OP_CAP_SLM_API_KEY');
|
|
44
|
-
|
|
45
|
-
for (const key of [...envKeys].sort()) {
|
|
46
|
-
lines.push(`${key}=`);
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
return lines.join('\n') + '\n';
|
|
50
|
-
}
|