@chatpanel/gateway 0.6.28 → 0.6.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/config.js +3 -6
- package/src/configstore.js +12 -8
- package/src/entitlement-refresh.js +8 -18
- package/src/entitlement.js +3 -8
- package/src/freegate.js +9 -18
- package/src/gateway-token.js +60 -0
- package/src/model-hashes.json +3 -0
- package/src/model-integrity.js +82 -0
- package/src/ner-engine.js +45 -6
- package/src/parakeet-engine.js +241 -0
- package/src/redact.js +5 -5
- package/src/router.js +4 -1
- package/src/secure-fetch.js +58 -0
- package/src/server.js +41 -18
- package/src/stt-engine.js +70 -4
- package/src/stt-models.js +27 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chatpanel/gateway",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.30",
|
|
4
4
|
"description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
"node": ">=18"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@chatpanel/pii": "^0.2.
|
|
30
|
+
"@chatpanel/pii": "^0.2.14",
|
|
31
31
|
"@huggingface/transformers": "^4.2.0",
|
|
32
32
|
"onnxruntime-web": "1.26.0-dev.20260416-b7804b056c"
|
|
33
33
|
},
|
package/src/config.js
CHANGED
|
@@ -34,12 +34,9 @@ const DEFAULTS = {
|
|
|
34
34
|
allowedOrigins: [],
|
|
35
35
|
maxBodyBytes: 26214400,
|
|
36
36
|
|
|
37
|
-
//
|
|
38
|
-
//
|
|
39
|
-
//
|
|
40
|
-
// ChatPanel Pro entitlement token (the same offline-signed token the extension/
|
|
41
|
-
// bridge use) to unlock unlimited usage. `free.used` is the running lifetime
|
|
42
|
-
// count (server-authoritative; persisted here so it survives restarts).
|
|
37
|
+
// `free.used` is the running lifetime redaction count for the free allowance
|
|
38
|
+
// (freegate.FREE_TOTAL_CAP redactions at full tier), persisted here. A valid
|
|
39
|
+
// ChatPanel entitlement token unlocks unlimited use.
|
|
43
40
|
pro: {
|
|
44
41
|
entitlementToken: '',
|
|
45
42
|
free: {
|
package/src/configstore.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// configure it live over the localhost API (GET/POST /config). The gateway stays
|
|
3
3
|
// authoritative — the extension is just a UI client.
|
|
4
4
|
|
|
5
|
-
import { writeFileSync, mkdirSync } from 'node:fs';
|
|
5
|
+
import { writeFileSync, mkdirSync, chmodSync } from 'node:fs';
|
|
6
6
|
import { join, dirname } from 'node:path';
|
|
7
7
|
import os from 'node:os';
|
|
8
8
|
import { usage } from './freegate.js';
|
|
@@ -15,7 +15,10 @@ export function configPath(env = process.env) {
|
|
|
15
15
|
|
|
16
16
|
// Persist the user-editable subset (not derived runtime state).
|
|
17
17
|
export function persistConfig(cfg, path = configPath()) {
|
|
18
|
-
|
|
18
|
+
// 0700 the dir + 0600 the file: this JSON holds per-destination apiKeys, the
|
|
19
|
+
// detector key, and the entitlement/bridge tokens — same secret-at-rest posture
|
|
20
|
+
// as the history key/secret files, so it isn't left world-readable on a shared host.
|
|
21
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
19
22
|
const out = {
|
|
20
23
|
host: cfg.host, port: cfg.port, backend: cfg.backend,
|
|
21
24
|
// Destinations (the configured agents + API models) MUST persist — otherwise a
|
|
@@ -25,7 +28,10 @@ export function persistConfig(cfg, path = configPath()) {
|
|
|
25
28
|
ner: cfg.ner, stt: cfg.stt, allowedOrigins: cfg.allowedOrigins, maxBodyBytes: cfg.maxBodyBytes,
|
|
26
29
|
pro: cfg.pro, logRequests: cfg.logRequests, logDetail: cfg.logDetail, tools: cfg.tools,
|
|
27
30
|
};
|
|
28
|
-
writeFileSync
|
|
31
|
+
// mode on writeFileSync only applies when CREATING the file; chmod after covers an
|
|
32
|
+
// already-existing (pre-fix, possibly 0644) config too. Best-effort for non-POSIX.
|
|
33
|
+
writeFileSync(path, JSON.stringify(out, null, 2), { mode: 0o600 });
|
|
34
|
+
try { chmodSync(path, 0o600); } catch { /* platforms without POSIX perms */ }
|
|
29
35
|
}
|
|
30
36
|
|
|
31
37
|
// Safe view for GET /config — never leak secrets (the entitlement + bridge tokens
|
|
@@ -47,8 +53,7 @@ export function publicConfig(cfg, { proUnlocked = false } = {}) {
|
|
|
47
53
|
ner: cfg.ner,
|
|
48
54
|
stt: cfg.stt,
|
|
49
55
|
allowedOrigins: Array.isArray(cfg.allowedOrigins) ? cfg.allowedOrigins : [],
|
|
50
|
-
// free = lifetime trial usage ({ used, cap, remaining })
|
|
51
|
-
// is fixed and the count is server-authoritative (never settable from the UI).
|
|
56
|
+
// free = lifetime trial usage ({ used, cap, remaining }), read-only.
|
|
52
57
|
pro: { unlocked: proUnlocked, hasToken: !!cfg.pro?.entitlementToken, free: usage(cfg) },
|
|
53
58
|
logRequests: !!cfg.logRequests,
|
|
54
59
|
logDetail: ['types', 'values'].includes(cfg.logDetail) ? cfg.logDetail : 'off',
|
|
@@ -106,9 +111,8 @@ export function applyConfigPatch(cfg, patch = {}) {
|
|
|
106
111
|
if (Array.isArray(patch.allowedOrigins)) cfg.allowedOrigins = patch.allowedOrigins;
|
|
107
112
|
if (patch.pro && typeof patch.pro === 'object') {
|
|
108
113
|
if (typeof patch.pro.entitlementToken === 'string') cfg.pro.entitlementToken = patch.pro.entitlementToken;
|
|
109
|
-
//
|
|
110
|
-
//
|
|
111
|
-
// client can't raise the cap or reset its own trial.
|
|
114
|
+
// The free allowance (freegate.FREE_TOTAL_CAP) and its `used` count are not
|
|
115
|
+
// part of the editable patch.
|
|
112
116
|
}
|
|
113
117
|
// Local dictation toggle. The MODEL is switched via POST /stt/models (like the
|
|
114
118
|
// NER manager), not patched here.
|
|
@@ -1,25 +1,15 @@
|
|
|
1
|
-
// Online entitlement re-validation
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// by re-polling the license worker; the gateway didn't, so a refund left unlimited
|
|
8
|
-
// redaction open for the rest of the token's life.
|
|
9
|
-
//
|
|
10
|
-
// So we also re-check ONLINE on an interval: poll the worker's /entitlement for the
|
|
11
|
-
// token's install_id and either
|
|
12
|
-
// • REFRESH the stored token (still entitled) — so it never lapses while paid, and
|
|
13
|
-
// • CLEAR it (worker says valid:false → refunded/cancelled/revoked) — dropping the
|
|
14
|
-
// gateway to Free immediately instead of riding the offline exp.
|
|
15
|
-
// Network/worker errors NEVER revoke (fail-open for paying users); the offline `exp`
|
|
16
|
-
// still bounds the worst case. Bounds post-refund Pro to <= CHECK_INTERVAL_MS.
|
|
1
|
+
// Online entitlement re-validation. The gateway stores one offline-signed
|
|
2
|
+
// entitlement token and verifies it offline (entitlement.js). On an interval it also
|
|
3
|
+
// re-checks online: it polls the worker's /entitlement for the token's install_id and
|
|
4
|
+
// either refreshes the stored token (still entitled) or clears it (worker reports
|
|
5
|
+
// valid:false), dropping the gateway to Free. Network/worker errors never revoke
|
|
6
|
+
// (fail-open); the offline `exp` still bounds the token's lifetime.
|
|
17
7
|
|
|
18
8
|
import { persistConfig, configPath } from './configstore.js';
|
|
19
9
|
|
|
20
10
|
// Same worker the extension/bridge use. Overridable for self-hosted/test.
|
|
21
11
|
const API_BASE = (process.env.CHATPANEL_API_BASE || 'https://api.chatpanel.net').replace(/\/+$/, '');
|
|
22
|
-
const CHECK_INTERVAL_MS = 60 * 60 * 1000; // 1h —
|
|
12
|
+
const CHECK_INTERVAL_MS = 60 * 60 * 1000; // 1h — online re-check interval
|
|
23
13
|
const FIRST_CHECK_DELAY_MS = 30 * 1000; // let the server settle before first poll
|
|
24
14
|
const MIN_RECHECK_MS = 2 * 60 * 1000; // throttle on-demand (/status) re-checks
|
|
25
15
|
|
|
@@ -63,7 +53,7 @@ async function revalidate(cfg) {
|
|
|
63
53
|
try { persistConfig(cfg, configPath()); } catch { /* best effort */ }
|
|
64
54
|
}
|
|
65
55
|
} else if (data && data.valid === false) {
|
|
66
|
-
//
|
|
56
|
+
// Worker reports the token is no longer valid → clear it and drop to Free.
|
|
67
57
|
cfg.pro.entitlementToken = '';
|
|
68
58
|
try { persistConfig(cfg, configPath()); } catch { /* best effort */ }
|
|
69
59
|
console.log('[gateway] entitlement no longer valid — Pro deactivated (refund/revoke/seat lost).');
|
package/src/entitlement.js
CHANGED
|
@@ -1,11 +1,6 @@
|
|
|
1
|
-
// Offline
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// The license server (Cloudflare Worker) signs a compact entitlement token with
|
|
5
|
-
// an ECDSA P-256 private key that lives ONLY there. The bridge ships the matching
|
|
6
|
-
// PUBLIC key and verifies the signature locally — no network, no secret. A forked
|
|
7
|
-
// client or a raw `curl` to the bridge can't forge entitlement without the
|
|
8
|
-
// private key, so this is a real cryptographic gate, not a UI check.
|
|
1
|
+
// Offline entitlement verification. The license server signs an ECDSA P-256 token;
|
|
2
|
+
// the gateway ships the matching public key and verifies it locally. Gates paid
|
|
3
|
+
// features such as custom "bring your own CLI" agents.
|
|
9
4
|
//
|
|
10
5
|
// Token format (identical to the extension's, extension/js/license.js):
|
|
11
6
|
// token = base64url(JSON payload) + "." + base64url(raw ECDSA signature)
|
package/src/freegate.js
CHANGED
|
@@ -1,20 +1,11 @@
|
|
|
1
|
-
// Free vs Pro for the gateway runtime
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// thing, then you buy. Pro (a valid ChatPanel entitlement token — the same
|
|
6
|
-
// offline-signed token the extension and bridge use) unlocks UNLIMITED redaction.
|
|
7
|
-
// The cryptographic check (entitlement.js) means a forked UI can't unlock it —
|
|
8
|
-
// only the configured/paid token does.
|
|
9
|
-
//
|
|
10
|
-
// The cap is LIFETIME (overall), not per-day, and it is NOT user-editable — so a
|
|
11
|
-
// free user gets exactly FREE_TOTAL_CAP genuine redactions, full stop. The count
|
|
12
|
-
// persists in cfg.pro.free.used (written via configstore.persistConfig), so it
|
|
13
|
-
// survives restarts.
|
|
1
|
+
// Free vs Pro for the gateway runtime. Without an entitlement token, full-tier
|
|
2
|
+
// redaction is available for a fixed lifetime number of redactions (FREE_TOTAL_CAP);
|
|
3
|
+
// a valid ChatPanel entitlement token unlocks unlimited redaction. The count is
|
|
4
|
+
// stored in cfg.pro.free.used (persisted via configstore.persistConfig).
|
|
14
5
|
|
|
15
6
|
import { isProEntitled } from './entitlement.js';
|
|
16
7
|
|
|
17
|
-
// The lifetime free allowance.
|
|
8
|
+
// The lifetime free allowance.
|
|
18
9
|
export const FREE_TOTAL_CAP = 100;
|
|
19
10
|
|
|
20
11
|
const proCache = { token: null, val: false };
|
|
@@ -34,10 +25,10 @@ export async function resolvePro(token) {
|
|
|
34
25
|
return val;
|
|
35
26
|
}
|
|
36
27
|
|
|
37
|
-
// May this request still redact? Pro = always
|
|
38
|
-
// cap is reached
|
|
39
|
-
//
|
|
40
|
-
//
|
|
28
|
+
// May this request still redact? Pro = always; Free = allowed until the lifetime
|
|
29
|
+
// cap is reached. This only CHECKS — consume() advances the count after a
|
|
30
|
+
// redaction actually happens, so requests with nothing to redact don't consume
|
|
31
|
+
// allowance.
|
|
41
32
|
export function checkQuota(cfg, isPro) {
|
|
42
33
|
if (isPro) return { allowed: true, remaining: Infinity, isPro: true };
|
|
43
34
|
const used = Number(cfg.pro?.free?.used) || 0;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// Per-install bearer token for the gateway's ADMIN routes (M2), mirroring the
|
|
2
|
+
// bridge's model. The gateway proxies model traffic for any local client (the /v1/*
|
|
3
|
+
// data plane stays open, no auth — that's the product), but its ADMIN surface
|
|
4
|
+
// (reconfigure via POST /config, read /logs) must not be reachable by every local
|
|
5
|
+
// process or by a drive-by localhost web page.
|
|
6
|
+
//
|
|
7
|
+
// Two ways to authenticate an admin call:
|
|
8
|
+
// • the ChatPanel extension — recognised by its chrome-/moz-extension:// Origin
|
|
9
|
+
// (a web page can't forge that; browsers set it honestly). The extension can't
|
|
10
|
+
// read a local file, so Origin is how IT authenticates.
|
|
11
|
+
// • a token — this 32-byte secret at ~/.chatpanel/gateway-token (0600), for a
|
|
12
|
+
// non-browser admin client (a setup script/CLI). A random local process doesn't
|
|
13
|
+
// have it.
|
|
14
|
+
// Residual (same as the bridge): a MALICIOUS local process could forge the
|
|
15
|
+
// extension Origin. A local attacker at that privilege level already owns much;
|
|
16
|
+
// this still blocks web drive-by and benign no-Origin processes from the admin API.
|
|
17
|
+
|
|
18
|
+
import { randomBytes, timingSafeEqual } from 'node:crypto';
|
|
19
|
+
import { existsSync, readFileSync, writeFileSync, mkdirSync, chmodSync } from 'node:fs';
|
|
20
|
+
import { join, dirname } from 'node:path';
|
|
21
|
+
import os from 'node:os';
|
|
22
|
+
|
|
23
|
+
const TOKEN_PATH = process.env.CHATPANEL_GATEWAY_TOKEN_PATH || join(os.homedir(), '.chatpanel', 'gateway-token');
|
|
24
|
+
let TOKEN = '';
|
|
25
|
+
|
|
26
|
+
// Load-or-create the token. Best-effort: token auth is hardening, never fail startup.
|
|
27
|
+
export function ensureGatewayToken(path = TOKEN_PATH) {
|
|
28
|
+
try {
|
|
29
|
+
if (existsSync(path)) TOKEN = readFileSync(path, 'utf8').trim();
|
|
30
|
+
if (!TOKEN) {
|
|
31
|
+
TOKEN = randomBytes(32).toString('hex');
|
|
32
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
33
|
+
writeFileSync(path, TOKEN, { mode: 0o600 });
|
|
34
|
+
try { chmodSync(path, 0o600); } catch { /* non-POSIX */ }
|
|
35
|
+
}
|
|
36
|
+
} catch (e) {
|
|
37
|
+
console.warn(`[gateway] could not initialise admin token: ${e?.message || e}`);
|
|
38
|
+
}
|
|
39
|
+
return TOKEN;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function isExtensionOrigin(origin) {
|
|
43
|
+
return typeof origin === 'string' && (/^chrome-extension:\/\//.test(origin) || /^moz-extension:\/\//.test(origin));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function tokenMatches(headers) {
|
|
47
|
+
if (!TOKEN) return false;
|
|
48
|
+
const h = String(headers?.authorization || '');
|
|
49
|
+
const provided = (h.startsWith('Bearer ') ? h.slice(7) : String(headers?.['x-chatpanel-token'] || '')).trim();
|
|
50
|
+
if (!provided) return false;
|
|
51
|
+
const a = Buffer.from(provided);
|
|
52
|
+
const b = Buffer.from(TOKEN);
|
|
53
|
+
return a.length === b.length && timingSafeEqual(a, b);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// True when a request may reach an ADMIN route: the extension (by Origin) or a
|
|
57
|
+
// token-bearing client.
|
|
58
|
+
export function isAdminAuthorized(req) {
|
|
59
|
+
return isExtensionOrigin(req.headers?.origin) || tokenMatches(req.headers);
|
|
60
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
{
|
|
2
|
+
"_README": "SHA-256 hashes of curated model weight files, keyed by modelId then by path relative to the model's cache dir (~/.chatpanel/models/<org>/<name>/). Generated on a TRUSTED host by tools/gen-model-hashes.mjs — do NOT hand-edit. Empty = verification is warn-and-allow (no protection yet); populate to enable fail-closed integrity checking. Keys starting with '_' are ignored by the loader."
|
|
3
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// Model-weight integrity verification (H3).
|
|
2
|
+
//
|
|
3
|
+
// The gateway downloads ONNX model weights (NER / Whisper / Parakeet) from the
|
|
4
|
+
// dl.chatpanel.net mirror or Hugging Face and loads them straight into an ONNX
|
|
5
|
+
// runtime. A compromised mirror, an HF repo takeover, or a MITM on a downgraded
|
|
6
|
+
// connection could therefore feed a malicious model into native code. This module
|
|
7
|
+
// verifies a downloaded model's weight files against committed SHA-256 hashes.
|
|
8
|
+
//
|
|
9
|
+
// Design (deliberately non-breaking):
|
|
10
|
+
// • Hashes live in the committed `model-hashes.json` — a { modelId: { relPath:
|
|
11
|
+
// sha256 } } map generated on a TRUSTED host by `tools/gen-model-hashes.mjs`.
|
|
12
|
+
// • FAIL-CLOSED only on a genuine mismatch: a listed file whose hash differs is
|
|
13
|
+
// deleted and the load is refused (throws). That's the security event.
|
|
14
|
+
// • WARN-AND-ALLOW everywhere else: a model not in the manifest, or a listed file
|
|
15
|
+
// not yet on disk, logs loudly and proceeds — so an empty/partial manifest never
|
|
16
|
+
// bricks model loading, and shipping the mechanism before the hashes are
|
|
17
|
+
// populated is safe. (Populate on a trusted machine to switch protection on.)
|
|
18
|
+
// • Verified on the DOWNLOAD path (when weights first arrive from the network —
|
|
19
|
+
// the supply-chain moment), not on every offline load (avoids re-hashing 100s of
|
|
20
|
+
// MB each boot).
|
|
21
|
+
|
|
22
|
+
import { createHash } from 'node:crypto';
|
|
23
|
+
import { readFileSync, existsSync, rmSync } from 'node:fs';
|
|
24
|
+
import { join } from 'node:path';
|
|
25
|
+
import { fileURLToPath } from 'node:url';
|
|
26
|
+
|
|
27
|
+
const MANIFEST_PATH = fileURLToPath(new URL('./model-hashes.json', import.meta.url));
|
|
28
|
+
let _manifest = null;
|
|
29
|
+
|
|
30
|
+
// { modelId: { 'onnx/model_quantized.onnx': '<sha256>' , … } }. Keys starting with
|
|
31
|
+
// '_' are metadata (schema note) and ignored. Missing/invalid file → empty map
|
|
32
|
+
// (verification simply becomes warn-and-allow for everything).
|
|
33
|
+
export function loadHashManifest(path = MANIFEST_PATH) {
|
|
34
|
+
if (_manifest && path === MANIFEST_PATH) return _manifest;
|
|
35
|
+
let parsed = {};
|
|
36
|
+
try {
|
|
37
|
+
const raw = JSON.parse(readFileSync(path, 'utf8'));
|
|
38
|
+
for (const [k, v] of Object.entries(raw)) {
|
|
39
|
+
if (!k.startsWith('_') && v && typeof v === 'object') parsed[k] = v;
|
|
40
|
+
}
|
|
41
|
+
} catch { parsed = {}; }
|
|
42
|
+
if (path === MANIFEST_PATH) _manifest = parsed;
|
|
43
|
+
return parsed;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function sha256File(path) {
|
|
47
|
+
return createHash('sha256').update(readFileSync(path)).digest('hex');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// Verify a model's weight files under `modelsDir`. Returns a result object; THROWS
|
|
51
|
+
// only on a real hash mismatch (fail-closed). `modelsDir` is the model cache root
|
|
52
|
+
// (e.g. ~/.chatpanel/models); the per-model dir is modelsDir/<org>/<name>.
|
|
53
|
+
/**
|
|
54
|
+
* @param {string} modelId
|
|
55
|
+
* @param {{ modelsDir?: string, log?: (msg: string) => void, manifest?: Record<string, Record<string, string>> }} [opts]
|
|
56
|
+
*/
|
|
57
|
+
export function verifyModelWeights(modelId, { modelsDir, log = () => {}, manifest = loadHashManifest() } = {}) {
|
|
58
|
+
const expected = manifest[modelId];
|
|
59
|
+
if (!expected || Object.keys(expected).length === 0) {
|
|
60
|
+
log(`[integrity] no recorded hashes for ${modelId} — skipping verification `
|
|
61
|
+
+ `(run tools/gen-model-hashes.mjs on a trusted host to enable)`);
|
|
62
|
+
return { verified: false, reason: 'unlisted', checked: 0 };
|
|
63
|
+
}
|
|
64
|
+
const dir = join(modelsDir, ...modelId.split('/'));
|
|
65
|
+
let checked = 0;
|
|
66
|
+
for (const [rel, wantSha] of Object.entries(expected)) {
|
|
67
|
+
const p = join(dir, ...rel.split('/'));
|
|
68
|
+
if (!existsSync(p)) { log(`[integrity] ${modelId}: listed file ${rel} not on disk — skipping`); continue; }
|
|
69
|
+
const gotSha = sha256File(p);
|
|
70
|
+
if (gotSha !== wantSha) {
|
|
71
|
+
try { rmSync(p); } catch { /* best effort */ }
|
|
72
|
+
throw new Error(
|
|
73
|
+
`[integrity] ${modelId}/${rel} SHA-256 mismatch `
|
|
74
|
+
+ `(expected ${wantSha.slice(0, 12)}…, got ${gotSha.slice(0, 12)}…) — `
|
|
75
|
+
+ `deleted the file and refusing to load a tampered model`,
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
checked++;
|
|
79
|
+
}
|
|
80
|
+
if (checked) log(`[integrity] ${modelId}: ${checked} weight file(s) verified ✓`);
|
|
81
|
+
return { verified: checked > 0, reason: checked ? 'ok' : 'no-files-present', checked };
|
|
82
|
+
}
|
package/src/ner-engine.js
CHANGED
|
@@ -18,15 +18,40 @@ import os from 'node:os';
|
|
|
18
18
|
import { join } from 'node:path';
|
|
19
19
|
import { existsSync, mkdirSync } from 'node:fs';
|
|
20
20
|
import { isKnownModel } from './models.js';
|
|
21
|
+
import { isLoopbackHost, isPrivateHost, isMetadataHost } from '@chatpanel/pii';
|
|
22
|
+
import { verifyModelWeights } from './model-integrity.js';
|
|
21
23
|
|
|
22
24
|
const DEFAULT_MODEL = 'Xenova/bert-base-NER';
|
|
25
|
+
const DEFAULT_MODEL_HOST = 'https://dl.chatpanel.net/models/';
|
|
23
26
|
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
|
|
27
|
+
// Where model weights are fetched from — ChatPanel's own edge-cached CDN by default,
|
|
28
|
+
// so a clean install depends only on chatpanel.net. CHATPANEL_MODEL_BASE_URL can
|
|
29
|
+
// override it (HF for dev, or an air-gapped LAN mirror) — but that env var is a
|
|
30
|
+
// download-redirect vector: an attacker who sets it could serve a malicious ONNX
|
|
31
|
+
// model into the runtime. So we VALIDATE the override before trusting it: http(s)
|
|
32
|
+
// only, never cloud metadata, and no PLAINTEXT http to a PUBLIC host (a LAN/loopback
|
|
33
|
+
// mirror on http is fine — that's the air-gap case). Anything else falls back to the
|
|
34
|
+
// signed default and logs loudly. (True per-file checksum verification is the
|
|
35
|
+
// remaining H3 step — needs a committed {model→sha256} manifest.)
|
|
36
|
+
export function resolveModelHost() {
|
|
37
|
+
const raw = process.env.CHATPANEL_MODEL_BASE_URL;
|
|
38
|
+
if (!raw) return DEFAULT_MODEL_HOST;
|
|
39
|
+
const fallback = (why) => {
|
|
40
|
+
console.warn(`[models] ignoring CHATPANEL_MODEL_BASE_URL (${raw}): ${why} — using ${DEFAULT_MODEL_HOST}`);
|
|
41
|
+
return DEFAULT_MODEL_HOST;
|
|
42
|
+
};
|
|
43
|
+
let u;
|
|
44
|
+
try { u = new URL(raw); } catch { return fallback('not a valid URL'); }
|
|
45
|
+
if (u.protocol !== 'http:' && u.protocol !== 'https:') return fallback(`scheme ${u.protocol} not allowed`);
|
|
46
|
+
if (isMetadataHost(u.hostname)) return fallback('points at cloud metadata');
|
|
47
|
+
const localish = isLoopbackHost(u.hostname) || isPrivateHost(u.hostname);
|
|
48
|
+
if (u.protocol === 'http:' && !localish) return fallback('plaintext http:// to a public host (use https or a LAN/loopback mirror)');
|
|
49
|
+
console.warn(`[models] ⚠ model weights will be downloaded from ${u.origin} (CHATPANEL_MODEL_BASE_URL override), not ${DEFAULT_MODEL_HOST}`);
|
|
50
|
+
return raw.replace(/\/*$/, '/');
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Must end with '/' (transformers appends the model path template to it).
|
|
54
|
+
const MODEL_HOST = resolveModelHost();
|
|
30
55
|
|
|
31
56
|
let _state = 'off'; // 'off' | 'loading' | 'downloading' | 'ready' | 'error'
|
|
32
57
|
let _model = null; // active model id, e.g. 'Xenova/bert-base-NER'
|
|
@@ -232,6 +257,20 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true } = {})
|
|
|
232
257
|
lib.env.allowRemoteModels = true;
|
|
233
258
|
pipe = await lib.pipeline('token-classification', modelId, { dtype: 'q8', progress_callback });
|
|
234
259
|
}
|
|
260
|
+
// H3: verify freshly-downloaded weights against committed hashes before we trust
|
|
261
|
+
// this pipeline. On a mismatch, dispose it (never use its outputs) and let the
|
|
262
|
+
// throw fall through to the catch (keeps the previous model / deterministic-only).
|
|
263
|
+
// NOTE: transformers downloads+loads in one call, so this runs POST-load — it
|
|
264
|
+
// prevents USING and PERSISTING a tampered model, not a hypothetical ORT
|
|
265
|
+
// parse-time bug. Warn-and-allow when the model isn't in the hash manifest.
|
|
266
|
+
if (!haveLocal) {
|
|
267
|
+
try {
|
|
268
|
+
verifyModelWeights(modelId, { modelsDir: modelRoot(), log });
|
|
269
|
+
} catch (e) {
|
|
270
|
+
try { await pipe.dispose?.(); } catch { /* ignore */ }
|
|
271
|
+
throw e;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
235
274
|
// Swap in the new pipeline, dispose the old one (free its WASM/native session).
|
|
236
275
|
_pipe = pipe; _model = modelId; _state = 'ready'; _err = null; _progress = null;
|
|
237
276
|
if (prevPipe && prevPipe !== pipe) { try { await prevPipe.dispose?.(); } catch { /* ignore */ } }
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
// In-process speech-to-text — NVIDIA Parakeet TDT (transducer) via onnxruntime.
|
|
2
|
+
//
|
|
3
|
+
// Whisper (stt-engine.js) is an encoder-DECODER seq2seq model and loads through the
|
|
4
|
+
// transformers.js `automatic-speech-recognition` pipeline. Parakeet is a *transducer*
|
|
5
|
+
// (Token-and-Duration Transducer / TDT on a FastConformer encoder) — a fundamentally
|
|
6
|
+
// different architecture the transformers.js pipeline can't drive (it errors with
|
|
7
|
+
// `Unsupported model type "nemo-conformer-tdt"`). So we run the ONNX graphs directly
|
|
8
|
+
// on the same onnxruntime the gateway already ships, with a hand-written greedy TDT
|
|
9
|
+
// decode loop.
|
|
10
|
+
//
|
|
11
|
+
// Why bother: Parakeet-TDT-0.6b-v3 is multilingual (25 European languages, auto-
|
|
12
|
+
// detected — no forced-language step) and runs ~35× realtime at int8 on CPU, several
|
|
13
|
+
// times faster than Whisper at comparable accuracy. It's the fast local-dictation path.
|
|
14
|
+
//
|
|
15
|
+
// Model layout (istupakov/onnx-asr export, e.g. `istupakov/parakeet-tdt-0.6b-v3-onnx`):
|
|
16
|
+
// nemo128.onnx mel preprocessor (waveforms → 128-bin log-mel features)
|
|
17
|
+
// encoder-model[.int8].onnx FastConformer encoder (features → [B,D,T'] frames)
|
|
18
|
+
// decoder_joint[.int8].onnx fused prediction-net (LSTM) + joint network
|
|
19
|
+
// vocab.txt "<piece> <id>" per line; the last line is "<blk> <id>"
|
|
20
|
+
//
|
|
21
|
+
// This module owns ONE concern: load those graphs and turn 16 kHz mono Float32 PCM
|
|
22
|
+
// into text. The streaming/session/redaction/diarization layer lives in stt-engine.js
|
|
23
|
+
// and calls us through a tiny adapter, so nothing downstream needs to know the engine.
|
|
24
|
+
|
|
25
|
+
import { join } from 'node:path';
|
|
26
|
+
import { existsSync, mkdirSync, readFileSync, createWriteStream, renameSync, statSync } from 'node:fs';
|
|
27
|
+
import { Readable } from 'node:stream';
|
|
28
|
+
import { modelRoot } from './ner-engine.js';
|
|
29
|
+
|
|
30
|
+
// transformers.js reports these `model_type`s for the transducer exports. Any of them
|
|
31
|
+
// means "not a whisper pipeline model — route here instead".
|
|
32
|
+
const TRANSDUCER_MODEL_TYPES = new Set([
|
|
33
|
+
'nemo-conformer-tdt', 'nemo-conformer-rnnt', 'parakeet_tdt', 'parakeet-tdt', 'parakeet_rnnt',
|
|
34
|
+
]);
|
|
35
|
+
export function isTransducerModelType(mt) {
|
|
36
|
+
return TRANSDUCER_MODEL_TYPES.has(String(mt || '').trim());
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Parakeet ships its own quantizations (standard QDQ/QOperator int8 — NOT whisper's
|
|
40
|
+
// block-quantized q8, so it loads on BOTH the native and WASM runtimes). Default to
|
|
41
|
+
// int8: ~690 MB total vs ~2.5 GB for fp32, with negligible accuracy loss for ASR.
|
|
42
|
+
export const PARAKEET_DEFAULT_DTYPE = 'int8';
|
|
43
|
+
export function parakeetDtype(d) {
|
|
44
|
+
return d === 'fp32' ? 'fp32' : PARAKEET_DEFAULT_DTYPE; // only int8 | fp32 are exported
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// The repo files this engine needs for a given precision. `encoder-model.onnx` (fp32)
|
|
48
|
+
// carries its weights in a sibling `.onnx.data` external-data file that ORT loads
|
|
49
|
+
// automatically when it sits next to the graph — so we must fetch it too.
|
|
50
|
+
function filesFor(dtype) {
|
|
51
|
+
const s = parakeetDtype(dtype) === 'fp32' ? '' : '.int8';
|
|
52
|
+
const files = ['config.json', 'vocab.txt', 'nemo128.onnx', `encoder-model${s}.onnx`, `decoder_joint-model${s}.onnx`];
|
|
53
|
+
if (!s) files.push('encoder-model.onnx.data'); // fp32 external weights
|
|
54
|
+
return files;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function parakeetDir(modelId) {
|
|
58
|
+
return join(modelRoot(), ...String(modelId).split('/'));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Present on disk = every required file exists and is non-empty (a truncated download
|
|
62
|
+
// must not read as "installed"). Mirrors stt/ner `modelOnDisk` intent.
|
|
63
|
+
export function parakeetOnDisk(modelId, dtype = PARAKEET_DEFAULT_DTYPE) {
|
|
64
|
+
const dir = parakeetDir(modelId);
|
|
65
|
+
if (!existsSync(dir)) return false;
|
|
66
|
+
try {
|
|
67
|
+
return filesFor(dtype).every((f) => { const p = join(dir, f); return existsSync(p) && statSync(p).size > 0; });
|
|
68
|
+
} catch { return false; }
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// ── onnxruntime, matching the gateway's runtime (native npm vs WASM binary) ──────────
|
|
72
|
+
// The npm gateway uses native onnxruntime-node (fast). The standalone binary embeds the
|
|
73
|
+
// onnxruntime-web WASM runtime and hands us its paths via __CHATPANEL_WASM_PATHS__ (the
|
|
74
|
+
// same global ner-engine keys off) — configure ORT-web from it. Memoized once.
|
|
75
|
+
let _ortPromise = null;
|
|
76
|
+
function getOrt() {
|
|
77
|
+
if (_ortPromise) return _ortPromise;
|
|
78
|
+
_ortPromise = (async () => {
|
|
79
|
+
const wasmPaths = globalThis.__CHATPANEL_WASM_PATHS__ || null;
|
|
80
|
+
const mod = await import(wasmPaths ? 'onnxruntime-web' : 'onnxruntime-node');
|
|
81
|
+
const ort = mod.InferenceSession ? mod : (mod.default || mod);
|
|
82
|
+
if (wasmPaths) {
|
|
83
|
+
try { ort.env.wasm.numThreads = 1; ort.env.wasm.proxy = false; ort.env.wasm.wasmPaths = wasmPaths; } catch { /* optional */ }
|
|
84
|
+
}
|
|
85
|
+
return ort;
|
|
86
|
+
})();
|
|
87
|
+
return _ortPromise;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// ── download (only when a model isn't already on disk) ───────────────────────────────
|
|
91
|
+
// Custom/BYO STT ids aren't on the dl.chatpanel.net mirror, so — like stt-engine's
|
|
92
|
+
// custom path — fetch straight from Hugging Face. Streamed to a .part file then renamed,
|
|
93
|
+
// so an interrupted download never looks complete. `onProgress({ file, pct })` drives
|
|
94
|
+
// the extension's model-manager UI.
|
|
95
|
+
async function downloadFile(modelId, file, dir, { onProgress, log } = {}) {
|
|
96
|
+
const url = `https://huggingface.co/${modelId}/resolve/main/${encodeURIComponent(file).replace(/%2F/g, '/')}`;
|
|
97
|
+
const res = await fetch(url, { redirect: 'follow' });
|
|
98
|
+
if (!res.ok || !res.body) throw new Error(`fetch ${file} → HTTP ${res.status}`);
|
|
99
|
+
const total = Number(res.headers.get('content-length')) || 0;
|
|
100
|
+
const tmp = join(dir, `${file}.part`);
|
|
101
|
+
const out = createWriteStream(tmp);
|
|
102
|
+
let got = 0, lastPct = -1;
|
|
103
|
+
const src = Readable.fromWeb(res.body);
|
|
104
|
+
src.on('data', (chunk) => {
|
|
105
|
+
got += chunk.length;
|
|
106
|
+
if (total) { const pct = Math.round((got / total) * 100); if (pct !== lastPct) { lastPct = pct; onProgress?.({ file, pct }); } }
|
|
107
|
+
});
|
|
108
|
+
await new Promise((resolve, reject) => {
|
|
109
|
+
src.pipe(out);
|
|
110
|
+
out.on('finish', resolve); out.on('error', reject); src.on('error', reject);
|
|
111
|
+
});
|
|
112
|
+
renameSync(tmp, join(dir, file));
|
|
113
|
+
log?.(`[parakeet] fetched ${file}`);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
async function ensureFiles(modelId, dtype, { onProgress, log } = {}) {
|
|
117
|
+
const dir = parakeetDir(modelId);
|
|
118
|
+
mkdirSync(dir, { recursive: true });
|
|
119
|
+
for (const file of filesFor(dtype)) {
|
|
120
|
+
const dest = join(dir, file);
|
|
121
|
+
if (existsSync(dest) && statSync(dest).size > 0) continue;
|
|
122
|
+
log?.(`[parakeet] downloading ${file}…`);
|
|
123
|
+
await downloadFile(modelId, file, dir, { onProgress, log });
|
|
124
|
+
}
|
|
125
|
+
return dir;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// ── vocab / detokenize ───────────────────────────────────────────────────────────────
|
|
129
|
+
// vocab.txt: "<piece> <id>" per line. SentencePiece marks a word boundary with ▁
|
|
130
|
+
// (U+2581); replace it with a space. The final line "<blk> <id>" is the blank/SOS id.
|
|
131
|
+
function loadVocab(dir) {
|
|
132
|
+
const vocab = [];
|
|
133
|
+
let blank = -1;
|
|
134
|
+
for (const line of readFileSync(join(dir, 'vocab.txt'), 'utf8').split('\n')) {
|
|
135
|
+
if (!line) continue;
|
|
136
|
+
const sp = line.lastIndexOf(' ');
|
|
137
|
+
if (sp < 0) continue;
|
|
138
|
+
const id = parseInt(line.slice(sp + 1), 10);
|
|
139
|
+
const tok = line.slice(0, sp);
|
|
140
|
+
if (!Number.isFinite(id)) continue;
|
|
141
|
+
vocab[id] = tok.replace(/▁/g, ' ');
|
|
142
|
+
if (tok === '<blk>') blank = id;
|
|
143
|
+
}
|
|
144
|
+
if (blank < 0) blank = vocab.length - 1; // fall back to the last id (export convention)
|
|
145
|
+
return { vocab, blank };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function argmax(arr, from, to) {
|
|
149
|
+
let bi = from, bv = arr[from];
|
|
150
|
+
for (let i = from + 1; i < to; i++) if (arr[i] > bv) { bv = arr[i]; bi = i; }
|
|
151
|
+
return bi;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const DURATIONS = 5; // TDT duration head bins → advance 0..4 encoder frames
|
|
155
|
+
const MAX_SYMBOLS = 10; // cap non-blank emissions per frame (anti-runaway)
|
|
156
|
+
|
|
157
|
+
// ── recognizer ───────────────────────────────────────────────────────────────────────
|
|
158
|
+
// Loads the three ONNX sessions once; `transcribe()` is stateless per call (fresh LSTM
|
|
159
|
+
// state), so it's safe to call for every streaming segment. Serialize calls externally
|
|
160
|
+
// (stt-engine already funnels decodes through one chain).
|
|
161
|
+
export async function loadRecognizer({ modelId, dtype = PARAKEET_DEFAULT_DTYPE, allowDownload = true, onProgress, log = () => {} }) {
|
|
162
|
+
const dt = parakeetDtype(dtype);
|
|
163
|
+
const onDisk = parakeetOnDisk(modelId, dt);
|
|
164
|
+
if (!onDisk && !allowDownload) throw new Error('model not on disk and downloads disabled');
|
|
165
|
+
const dir = onDisk ? parakeetDir(modelId) : await ensureFiles(modelId, dt, { onProgress, log });
|
|
166
|
+
|
|
167
|
+
const ort = await getOrt();
|
|
168
|
+
const s = dt === 'fp32' ? '' : '.int8';
|
|
169
|
+
const opts = { executionProviders: ['cpu'], graphOptimizationLevel: 'all', logSeverityLevel: 3 };
|
|
170
|
+
const [prep, encoder, decoder] = await Promise.all([
|
|
171
|
+
ort.InferenceSession.create(join(dir, 'nemo128.onnx'), opts),
|
|
172
|
+
ort.InferenceSession.create(join(dir, `encoder-model${s}.onnx`), opts),
|
|
173
|
+
ort.InferenceSession.create(join(dir, `decoder_joint-model${s}.onnx`), opts),
|
|
174
|
+
]);
|
|
175
|
+
const { vocab, blank } = loadVocab(dir);
|
|
176
|
+
const VOCAB = blank + 1; // token logits span [0, VOCAB); duration logits follow
|
|
177
|
+
const Tensor = ort.Tensor;
|
|
178
|
+
|
|
179
|
+
async function transcribe(float32) {
|
|
180
|
+
if (!(float32 instanceof Float32Array) || float32.length < 400) return '';
|
|
181
|
+
const n = float32.length;
|
|
182
|
+
|
|
183
|
+
// 1) mel features (done in-graph — no manual DSP).
|
|
184
|
+
const pr = await prep.run({
|
|
185
|
+
waveforms: new Tensor('float32', float32, [1, n]),
|
|
186
|
+
waveforms_lens: new Tensor('int64', BigInt64Array.from([BigInt(n)]), [1]),
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
// 2) FastConformer encoder → outputs [1, D, T'] (channels-first, TIME LAST).
|
|
190
|
+
const er = await encoder.run({ audio_signal: pr.features, length: pr.features_lens });
|
|
191
|
+
const enc = er.outputs;
|
|
192
|
+
const [, D, T] = enc.dims;
|
|
193
|
+
const encLen = Number(er.encoded_lengths.data[0]);
|
|
194
|
+
const ed = enc.data; // element (0,d,t) at index d*T + t
|
|
195
|
+
|
|
196
|
+
// 3) greedy TDT decode. Per encoder frame: run the fused prednet+joint on the
|
|
197
|
+
// previous token + LSTM state; split the logits into token- and duration-heads.
|
|
198
|
+
// Only a NON-BLANK emission appends a token and advances the LSTM state; the
|
|
199
|
+
// duration argmax says how many frames to jump (0..4). duration==0 lets us emit
|
|
200
|
+
// another symbol at the same frame (up to MAX_SYMBOLS) — this is what makes TDT
|
|
201
|
+
// faster than plain RNN-T.
|
|
202
|
+
let h = new Float32Array(2 * 640);
|
|
203
|
+
let c = new Float32Array(2 * 640);
|
|
204
|
+
const tokens = [];
|
|
205
|
+
const frame = new Float32Array(D);
|
|
206
|
+
let t = 0, emitted = 0;
|
|
207
|
+
const guard = encLen * (MAX_SYMBOLS + 1) + 8; // hard stop; the loop always advances, but be safe
|
|
208
|
+
for (let iter = 0; t < encLen && iter < guard; iter++) {
|
|
209
|
+
for (let d = 0; d < D; d++) frame[d] = ed[d * T + t];
|
|
210
|
+
const prev = tokens.length ? tokens[tokens.length - 1] : blank;
|
|
211
|
+
const out = await decoder.run({
|
|
212
|
+
encoder_outputs: new Tensor('float32', frame, [1, D, 1]),
|
|
213
|
+
targets: new Tensor('int32', Int32Array.from([prev]), [1, 1]),
|
|
214
|
+
target_length: new Tensor('int32', Int32Array.from([1]), [1]),
|
|
215
|
+
input_states_1: new Tensor('float32', h, [2, 1, 640]),
|
|
216
|
+
input_states_2: new Tensor('float32', c, [2, 1, 640]),
|
|
217
|
+
});
|
|
218
|
+
const logits = out.outputs.data;
|
|
219
|
+
const tok = argmax(logits, 0, VOCAB);
|
|
220
|
+
const durIdx = argmax(logits, VOCAB, VOCAB + DURATIONS) - VOCAB; // 0..4
|
|
221
|
+
if (tok !== blank) {
|
|
222
|
+
h = out.output_states_1.data; c = out.output_states_2.data; // advance state only on emit
|
|
223
|
+
tokens.push(tok);
|
|
224
|
+
emitted++;
|
|
225
|
+
}
|
|
226
|
+
if (durIdx > 0) { t += durIdx; emitted = 0; }
|
|
227
|
+
else if (tok === blank || emitted >= MAX_SYMBOLS) { t += 1; emitted = 0; }
|
|
228
|
+
// else duration==0 & non-blank & under cap → stay on this frame, emit again
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
let text = '';
|
|
232
|
+
for (const id of tokens) text += vocab[id] ?? '';
|
|
233
|
+
return text.replace(/^\s+/, '').replace(/\s+/g, ' ').trimEnd();
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function dispose() {
|
|
237
|
+
for (const sess of [prep, encoder, decoder]) { try { sess.release?.(); } catch { /* ignore */ } }
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
return { transcribe, dispose, dtype: dt, dir };
|
|
241
|
+
}
|
package/src/redact.js
CHANGED
|
@@ -14,10 +14,10 @@ import * as engine from './ner-engine.js';
|
|
|
14
14
|
// tier: 'basic' | 'full'. For 'full' we run the local detector over the combined
|
|
15
15
|
// text to harvest names/orgs, then redact every segment against that entity set.
|
|
16
16
|
//
|
|
17
|
-
// Free
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
17
|
+
// Free use is bounded by a request quota (freegate.js), not by downgrading
|
|
18
|
+
// quality — free requests get the full tier (names/orgs via NER) within their
|
|
19
|
+
// allowance. The custom dictionary is capped for free: gatedDictionary limits it
|
|
20
|
+
// to FREE_DICT_LIMIT via the shared chatpanel-pii gate.
|
|
21
21
|
export async function redactSegments(segments, redactionCfg, { signal, isPro = true } = {}) {
|
|
22
22
|
const vault = createVault();
|
|
23
23
|
|
|
@@ -38,7 +38,7 @@ export async function redactSegments(segments, redactionCfg, { signal, isPro = t
|
|
|
38
38
|
const texts = segments.map((s) => s.get()).filter((t) => typeof t === 'string' && t);
|
|
39
39
|
if (texts.length === 0) return { vault, count: 0, sanitized };
|
|
40
40
|
|
|
41
|
-
// Use the configured tier as-is (no free downgrade — the quota
|
|
41
|
+
// Use the configured tier as-is (no free downgrade — the quota bounds free use),
|
|
42
42
|
// but keep the dictionary capped for free via the shared chatpanel-pii gate.
|
|
43
43
|
const tier = redactionCfg.tier === 'full' ? 'full' : 'basic';
|
|
44
44
|
const dictionary = gatedDictionary(redactionCfg, isPro);
|
package/src/router.js
CHANGED
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
// baseUrl, protocol, (api) where to forward + 'openai'|'anthropic'
|
|
11
11
|
// models: [..], models this destination serves (for /v1/models)
|
|
12
12
|
// }
|
|
13
|
+
|
|
14
|
+
import { secureFetch } from './secure-fetch.js';
|
|
13
15
|
//
|
|
14
16
|
// /v1/models aggregates every destination's models so clients can discover them.
|
|
15
17
|
|
|
@@ -87,7 +89,8 @@ export async function aggregateModelsAsync(cfg, { timeoutMs = 4000 } = {}) {
|
|
|
87
89
|
if (d.protocol === 'anthropic') { headers['x-api-key'] = d.apiKey; headers['anthropic-version'] = '2023-06-01'; }
|
|
88
90
|
else headers.authorization = `Bearer ${d.apiKey}`;
|
|
89
91
|
}
|
|
90
|
-
|
|
92
|
+
// secureFetch: SSRF guard (scheme/host + resolved-IP); a blocked dest throws → skipped via the catch.
|
|
93
|
+
const res = await secureFetch(`${d.baseUrl.replace(/\/$/, '')}/models`, { headers, signal: ctrl.signal });
|
|
91
94
|
if (!res.ok) return;
|
|
92
95
|
const j = await res.json();
|
|
93
96
|
const list = Array.isArray(j?.data) ? j.data : (Array.isArray(j?.models) ? j.models : []);
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// Node-side SSRF-hardened fetch (M1). The shared @chatpanel/pii/net.js classifier is
|
|
2
|
+
// browser-safe and can only match on the URL's HOSTNAME string — so a public name
|
|
3
|
+
// whose A/AAAA record resolves to a private/metadata IP (DNS-rebinding at the fetch
|
|
4
|
+
// layer) slips past it. Node CAN resolve DNS, so here we additionally validate every
|
|
5
|
+
// RESOLVED address against the same policy, and revalidate on each redirect hop.
|
|
6
|
+
//
|
|
7
|
+
// Policy defaults to the ENDPOINT context (loopback + LAN allowed — Ollama / a LAN
|
|
8
|
+
// model box are legitimate gateway upstreams — but cloud metadata never). So in
|
|
9
|
+
// practice the DNS check blocks a hostname that resolves to cloud metadata; the
|
|
10
|
+
// redirect revalidation blocks a public upstream that 3xx-redirects to a blocked host.
|
|
11
|
+
//
|
|
12
|
+
// Residual (documented): we validate the resolved IPs but do NOT pin the socket to
|
|
13
|
+
// them, so a sub-second TOCTOU window between lookup and connect remains. Closing it
|
|
14
|
+
// fully needs a custom undici dispatcher; the resolved-IP check already removes the
|
|
15
|
+
// easy DNS-rebinding cases, which is the point of M1.
|
|
16
|
+
|
|
17
|
+
import { lookup as dnsLookup } from 'node:dns/promises';
|
|
18
|
+
import { assertEndpointUrl, isBlockedHost } from '@chatpanel/pii';
|
|
19
|
+
|
|
20
|
+
const ENDPOINT_POLICY = { allowLoopback: true, allowPrivate: true };
|
|
21
|
+
const isLiteralIp = (h) => /^\d{1,3}(\.\d{1,3}){3}$/.test(h) || h.includes(':');
|
|
22
|
+
|
|
23
|
+
// Reject if ANY resolved address is blocked under `policy`. A literal-IP host is
|
|
24
|
+
// already covered by the URL check, so skip DNS for it. A resolution failure is left
|
|
25
|
+
// for fetch() to surface (don't mask a real network error as a security error).
|
|
26
|
+
async function assertResolvedIps(hostname, policy, lookupFn) {
|
|
27
|
+
if (isLiteralIp(hostname)) return;
|
|
28
|
+
let addrs;
|
|
29
|
+
try { addrs = await lookupFn(hostname, { all: true }); } catch { return; }
|
|
30
|
+
for (const { address } of addrs) {
|
|
31
|
+
if (isBlockedHost(address, policy)) {
|
|
32
|
+
throw new Error(`refusing to reach ${hostname} — it resolves to a blocked address (${address})`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// secureFetch(url, { policy?, maxRedirects?, lookupFn?, fetchFn?, ...init })
|
|
38
|
+
// Validates the URL (scheme + host policy) AND its resolved IPs before each request,
|
|
39
|
+
// following redirects manually so every hop is re-checked. Returns the final Response.
|
|
40
|
+
// `...init` forwards any standard fetch options (method, headers, body, signal, …).
|
|
41
|
+
/**
|
|
42
|
+
* @param {string|URL} url
|
|
43
|
+
* @param {any} [opts]
|
|
44
|
+
*/
|
|
45
|
+
export async function secureFetch(url, {
|
|
46
|
+
policy = ENDPOINT_POLICY, maxRedirects = 5, lookupFn = dnsLookup, fetchFn = fetch, ...init
|
|
47
|
+
} = {}) {
|
|
48
|
+
let current = assertEndpointUrl(url, policy);
|
|
49
|
+
for (let hop = 0; ; hop++) {
|
|
50
|
+
await assertResolvedIps(current.hostname, policy, lookupFn);
|
|
51
|
+
const res = await fetchFn(current.toString(), { ...init, redirect: 'manual' });
|
|
52
|
+
const loc = res.status >= 300 && res.status < 400 ? res.headers.get('location') : null;
|
|
53
|
+
if (!loc) return res;
|
|
54
|
+
if (hop >= maxRedirects) throw new Error('too many redirects');
|
|
55
|
+
current = assertEndpointUrl(new URL(loc, current).toString(), policy); // revalidate the hop
|
|
56
|
+
try { res.body?.cancel?.(); } catch { /* free the redirect body */ }
|
|
57
|
+
}
|
|
58
|
+
}
|
package/src/server.js
CHANGED
|
@@ -22,7 +22,9 @@ import { loadConfig } from './config.js';
|
|
|
22
22
|
import { startEntitlementRefresh, maybeRevalidate } from './entitlement-refresh.js';
|
|
23
23
|
import { redactSegments, segment } from './redact.js';
|
|
24
24
|
import { pipeRestoredStream, pipeRestoredOpenAIStream, makeTokenRestorer } from './stream.js';
|
|
25
|
-
import { restoreText, gatedDictionary, narrowSpecs, makeToolHarness, placeholderToolNote } from '@chatpanel/pii';
|
|
25
|
+
import { restoreText, gatedDictionary, narrowSpecs, makeToolHarness, placeholderToolNote, assertEndpointUrl } from '@chatpanel/pii';
|
|
26
|
+
import { ensureGatewayToken, isAdminAuthorized } from './gateway-token.js';
|
|
27
|
+
import { secureFetch } from './secure-fetch.js';
|
|
26
28
|
import { streamBridgeChat, readBridgeToken, openBridgeChat } from './bridge.js';
|
|
27
29
|
import { createRelaySession, getRelaySession, endRelaySession, pumpBridgeStream, deliverToolResult, toolsToSpecs, parseToolCallId } from './toolrelay.js';
|
|
28
30
|
import { shaperFor } from './shape.js';
|
|
@@ -43,7 +45,7 @@ import * as openai from './openai.js';
|
|
|
43
45
|
import * as responses from './responses.js';
|
|
44
46
|
import * as anthropic from './anthropic.js';
|
|
45
47
|
|
|
46
|
-
export const VERSION = '0.6.
|
|
48
|
+
export const VERSION = '0.6.30';
|
|
47
49
|
|
|
48
50
|
// WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
|
|
49
51
|
// store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
|
|
@@ -255,7 +257,8 @@ async function probeNerHealth(cfg) {
|
|
|
255
257
|
const url = nerBaseUrl(cfg);
|
|
256
258
|
if (!url) return { configured: false, ok: false, url: null, model: null };
|
|
257
259
|
try {
|
|
258
|
-
|
|
260
|
+
// secureFetch: scheme + host policy AND resolved-IP validation (DNS-rebinding).
|
|
261
|
+
const r = await secureFetch(url.replace(/\/ner\/?$/, '') + '/health', { signal: AbortSignal.timeout(2000) });
|
|
259
262
|
if (!r.ok) return { configured: true, ok: false, url, model: null };
|
|
260
263
|
const j = await r.json().catch(() => ({}));
|
|
261
264
|
return { configured: true, ok: true, url, model: j.model || null };
|
|
@@ -306,8 +309,8 @@ async function startRelay(req, res, { kind, adapter, agent }, body, vault, cfg,
|
|
|
306
309
|
const { messages, system } = adapter.toTurn(body);
|
|
307
310
|
const token = readBridgeToken(cfg.bridge.token);
|
|
308
311
|
const shaper = shaperFor(kind, body?.model || agent);
|
|
309
|
-
// Full tier for everyone here (the free allowance is enforced
|
|
310
|
-
//
|
|
312
|
+
// Full tier for everyone here (the free allowance is enforced in the main
|
|
313
|
+
// handler), but the custom dictionary stays capped for free.
|
|
311
314
|
const redactOpts = { tier: cfg.redaction.tier === 'full' ? 'full' : 'basic', dictionary: gatedDictionary(cfg.redaction, isPro), entities: [] };
|
|
312
315
|
const s = createRelaySession({ vault, redactOpts, bridgeUrl: cfg.bridge.url, token, harness });
|
|
313
316
|
const ttl = setTimeout(() => endRelaySession(s.id), 135_000); // bridge tool-call timeout is 120s
|
|
@@ -425,7 +428,11 @@ async function handleApi(req, res, { adapter, kind, pathname, search, base, dest
|
|
|
425
428
|
if (destProtocol === 'anthropic') { headers['x-api-key'] = destKey; delete headers.authorization; }
|
|
426
429
|
else { headers.authorization = `Bearer ${destKey}`; }
|
|
427
430
|
}
|
|
428
|
-
|
|
431
|
+
// SSRF guard on the config-supplied upstream: block cloud-metadata + non-http(s)
|
|
432
|
+
// BEFORE the fetch. Loopback/LAN stay allowed (Ollama/LM Studio/homelab are the
|
|
433
|
+
// point of a BYO gateway); only the credential-theft pivot is refused.
|
|
434
|
+
const upstreamUrl = assertEndpointUrl(base.replace(/\/$/, '') + pathname + search).toString();
|
|
435
|
+
upstream = await fetch(upstreamUrl, {
|
|
429
436
|
method: req.method,
|
|
430
437
|
headers,
|
|
431
438
|
body: ['GET', 'HEAD'].includes(req.method) ? undefined : outBody,
|
|
@@ -492,6 +499,14 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
492
499
|
if (req.headers.origin) setCors(res, req.headers.origin);
|
|
493
500
|
if (req.method === 'OPTIONS') { res.writeHead(204); return res.end(); }
|
|
494
501
|
|
|
502
|
+
// M2: ADMIN routes reconfigure the gateway (POST /config) or expose its in-memory
|
|
503
|
+
// logs (GET /logs). Unlike the /v1 data plane (open to any local client — the
|
|
504
|
+
// product), these must not be reachable by a no-Origin local process or a drive-by
|
|
505
|
+
// localhost web page. Require the extension Origin or the gateway token.
|
|
506
|
+
if ((pathname === '/config' || pathname === '/logs') && !isAdminAuthorized(req)) {
|
|
507
|
+
return sendJson(res, 403, { error: 'admin route: extension origin or gateway token required' });
|
|
508
|
+
}
|
|
509
|
+
|
|
495
510
|
if (req.method === 'GET' && pathname === '/health') {
|
|
496
511
|
// `stt` is ADDITIVE (Tesla rule): old clients ignore it, new clients use it
|
|
497
512
|
// to auto-detect local dictation. `enabled` reflects config; the model only
|
|
@@ -621,7 +636,8 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
621
636
|
if (!url) return sendJson(res, 503, { error: { message: 'NER not configured — deterministic-only redaction', type: 'ner_off' } });
|
|
622
637
|
try {
|
|
623
638
|
const body = await readBody(req, cfg.maxBodyBytes);
|
|
624
|
-
|
|
639
|
+
// secureFetch: scheme/host policy + resolved-IP check before POSTing raw text to the detector.
|
|
640
|
+
const r = await secureFetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body, signal: AbortSignal.timeout(8000) });
|
|
625
641
|
const text = await r.text();
|
|
626
642
|
res.writeHead(r.status, { 'content-type': 'application/json' });
|
|
627
643
|
return res.end(text);
|
|
@@ -892,9 +908,9 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
892
908
|
body.tools = narrowSpecs(body.tools, latestUserText(body, r.kind), { cap, keep, name: toolName, description: toolDesc });
|
|
893
909
|
narrowedTools = before - body.tools.length;
|
|
894
910
|
}
|
|
895
|
-
// Free
|
|
896
|
-
//
|
|
897
|
-
//
|
|
911
|
+
// Free users get full-tier redaction within a fixed lifetime allowance —
|
|
912
|
+
// checked here, consumed below once a redaction actually happens. Over the
|
|
913
|
+
// cap returns 402.
|
|
898
914
|
isPro = await resolvePro(cfg.pro?.entitlementToken);
|
|
899
915
|
const allow = checkQuota(cfg, isPro);
|
|
900
916
|
if (!allow.allowed) {
|
|
@@ -907,16 +923,16 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
907
923
|
const ac = new AbortController();
|
|
908
924
|
req.on('close', () => ac.abort());
|
|
909
925
|
const rd0 = trace ? trace.clock() : 0;
|
|
910
|
-
// Redact at the configured tier for everyone (free users get
|
|
911
|
-
//
|
|
912
|
-
//
|
|
926
|
+
// Redact at the configured tier for everyone (free users get name/org
|
|
927
|
+
// redaction within their allowance); the custom dictionary stays capped for
|
|
928
|
+
// free (isPro decides that inside).
|
|
913
929
|
const { vault: v, count, sanitized } = await redactSegments(segs, cfg.redaction, { signal: ac.signal, isPro });
|
|
914
930
|
if (trace) trace.lap('redact', rd0);
|
|
915
931
|
vault = v;
|
|
916
932
|
redactedCount = count;
|
|
917
933
|
sanitizedCount = sanitized || 0;
|
|
918
|
-
//
|
|
919
|
-
// then persist
|
|
934
|
+
// Consume one lifetime free credit only when we actually redacted
|
|
935
|
+
// something, then persist. (No-op / no write for Pro.)
|
|
920
936
|
if (!isPro && count > 0) {
|
|
921
937
|
consume(cfg, isPro);
|
|
922
938
|
try { persistConfig(cfg, configPath()); } catch { /* best effort — usage is advisory */ }
|
|
@@ -958,11 +974,12 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
958
974
|
|
|
959
975
|
export function start(cfg = loadConfig()) {
|
|
960
976
|
installTimestampedConsole(); // every gateway log line gets a clock — before anything logs
|
|
977
|
+
ensureGatewayToken(); // M2: load/create the admin-route token (best-effort)
|
|
961
978
|
const server = createGateway(cfg);
|
|
962
979
|
const ner = startNer(cfg); // may mutate cfg.redaction when it comes up
|
|
963
|
-
// Re-validate the stored
|
|
964
|
-
//
|
|
965
|
-
//
|
|
980
|
+
// Re-validate the stored entitlement online on an interval; clears it and drops
|
|
981
|
+
// the gateway to Free when the worker reports it invalid (see
|
|
982
|
+
// entitlement-refresh.js).
|
|
966
983
|
const entitlement = startEntitlementRefresh(cfg);
|
|
967
984
|
// Fail LOUD on a port clash instead of crashing with a raw stack trace. We bind a
|
|
968
985
|
// FIXED port (4320) so the extension / install.sh / OpenCode can always find us; if
|
|
@@ -983,6 +1000,12 @@ export function start(cfg = loadConfig()) {
|
|
|
983
1000
|
console.log(` backend : ${cfg.backend}` + (cfg.backend === 'bridge' ? ` (agent: ${cfg.bridge.agent}, via ${cfg.bridge.url})` : ''));
|
|
984
1001
|
console.log(` redaction: ${cfg.redaction.tier}` + (cfg.redaction.detection?.backend && cfg.redaction.detection.backend !== 'off'
|
|
985
1002
|
? ` + ${cfg.redaction.detection.backend} detector` : (cfg.ner?.autostart ? ' (+ NER starting…)' : '')));
|
|
1003
|
+
// M7: a non-loopback bind exposes the gateway on the LAN, where the per-request
|
|
1004
|
+
// loopback Host check is trivially satisfied by a spoofed `Host: 127.0.0.1`. The
|
|
1005
|
+
// /v1 data plane forwards with the client's own key, but make the exposure LOUD.
|
|
1006
|
+
if (!isLoopbackHost(cfg.host)) {
|
|
1007
|
+
console.error(`⚠ SECURITY: gateway bound to NON-LOOPBACK host ${cfg.host}. It is reachable off-machine, and the loopback Host-header check is spoofable from the LAN. Admin routes still need the token/extension, but prefer binding 127.0.0.1 unless you intend LAN exposure on a trusted network.`);
|
|
1008
|
+
}
|
|
986
1009
|
});
|
|
987
1010
|
// If the user handed off a backup key, refresh the warm store from the latest
|
|
988
1011
|
// daily backup in the background — so the gateway stays current even when the
|
package/src/stt-engine.js
CHANGED
|
@@ -20,7 +20,9 @@ import { join } from 'node:path';
|
|
|
20
20
|
import { existsSync, readdirSync } from 'node:fs';
|
|
21
21
|
import { randomUUID } from 'node:crypto';
|
|
22
22
|
import { ensureLib, modelRoot } from './ner-engine.js';
|
|
23
|
-
import {
|
|
23
|
+
import { verifyModelWeights } from './model-integrity.js';
|
|
24
|
+
import { DEFAULT_STT_MODEL, isEnglishOnly, sttModelDtype, isKnownSttModel, sttModelEngine } from './stt-models.js';
|
|
25
|
+
import * as parakeet from './parakeet-engine.js';
|
|
24
26
|
import * as diarize from './diarize-engine.js';
|
|
25
27
|
|
|
26
28
|
export const SAMPLE_RATE = 16000; // fixed wire contract: 16 kHz mono Float32 PCM
|
|
@@ -65,6 +67,8 @@ const DTYPE_SUFFIX = {
|
|
|
65
67
|
};
|
|
66
68
|
|
|
67
69
|
export function modelOnDisk(modelId = _model || DEFAULT_STT_MODEL, dtype = sttModelDtype(modelId) || runtimeDtype()) {
|
|
70
|
+
// Transducer models (parakeet) have a different file layout + engine — delegate.
|
|
71
|
+
if (sttModelEngine(modelId) === 'parakeet-tdt') return parakeet.parakeetOnDisk(modelId, parakeet.parakeetDtype(dtype));
|
|
68
72
|
const dir = join(modelRoot(), ...modelId.split('/'), 'onnx');
|
|
69
73
|
if (!existsSync(dir)) return false;
|
|
70
74
|
const suffix = DTYPE_SUFFIX[dtype] ?? '';
|
|
@@ -87,6 +91,9 @@ export function health() {
|
|
|
87
91
|
// and a failed SWITCH keeps the previous working pipeline.
|
|
88
92
|
/** @param {string} modelId @param {{ log?: (m: string) => void, allowDownload?: boolean, dtype?: string }} [opts] */
|
|
89
93
|
async function loadModel(modelId, { log = () => {}, allowDownload = true, dtype: dtypeOverride = null } = {}) {
|
|
94
|
+
// Transducer models aren't whisper pipelines — hand off to the parakeet engine.
|
|
95
|
+
if (sttModelEngine(modelId) === 'parakeet-tdt') return loadParakeet(modelId, { log, allowDownload, dtype: dtypeOverride });
|
|
96
|
+
|
|
90
97
|
const prevPipe = _pipe;
|
|
91
98
|
const prevModel = _model;
|
|
92
99
|
let lib;
|
|
@@ -145,6 +152,16 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true, dtype:
|
|
|
145
152
|
lib.env.allowRemoteModels = true;
|
|
146
153
|
pipe = await lib.pipeline('automatic-speech-recognition', modelId, { dtype, progress_callback });
|
|
147
154
|
}
|
|
155
|
+
// H3: verify freshly-downloaded weights (fail-closed on a real mismatch;
|
|
156
|
+
// warn-and-allow when unlisted). Post-load — see model-integrity.js note.
|
|
157
|
+
if (!haveLocal) {
|
|
158
|
+
try {
|
|
159
|
+
verifyModelWeights(modelId, { modelsDir: modelRoot(), log });
|
|
160
|
+
} catch (e) {
|
|
161
|
+
try { await pipe.dispose?.(); } catch { /* ignore */ }
|
|
162
|
+
throw e;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
148
165
|
_pipe = pipe; _model = modelId; _state = 'ready'; _err = null; _progress = null; _dtype = dtype;
|
|
149
166
|
if (prevPipe && prevPipe !== pipe) { try { await prevPipe.dispose?.(); } catch { /* ignore */ } }
|
|
150
167
|
log(`[stt] ready — model ${modelId} @ ${dtype} (in-process, offline) — local dictation active`);
|
|
@@ -160,6 +177,52 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true, dtype:
|
|
|
160
177
|
}
|
|
161
178
|
}
|
|
162
179
|
|
|
180
|
+
// Load a Parakeet TDT (transducer) model via parakeet-engine.js (raw onnxruntime), and
|
|
181
|
+
// expose it to the session layer as a whisper-shaped `_pipe(audio) → { text }` adapter,
|
|
182
|
+
// so decodeSession/streaming/redaction/diarization all work unchanged. Fail-open and
|
|
183
|
+
// keep-previous-on-switch-failure, exactly like the whisper path above.
|
|
184
|
+
/** @param {string} modelId @param {{ log?: (m: string) => void, allowDownload?: boolean, dtype?: string|null }} [opts] */
|
|
185
|
+
async function loadParakeet(modelId, { log = () => {}, allowDownload = true, dtype: dtypeOverride = null } = {}) {
|
|
186
|
+
const prevPipe = _pipe;
|
|
187
|
+
const prevModel = _model;
|
|
188
|
+
const dtype = parakeet.parakeetDtype(dtypeOverride && dtypeOverride !== 'auto' ? dtypeOverride : PARAKEET_RUNTIME_DTYPE());
|
|
189
|
+
const haveLocal = parakeet.parakeetOnDisk(modelId, dtype);
|
|
190
|
+
if (!haveLocal && !allowDownload) {
|
|
191
|
+
_state = 'error'; _err = 'model not on disk and downloads disabled';
|
|
192
|
+
log(`[stt] model ${modelId} not installed and downloads disabled`);
|
|
193
|
+
return false;
|
|
194
|
+
}
|
|
195
|
+
_state = haveLocal ? 'loading' : 'downloading';
|
|
196
|
+
if (!haveLocal) { _progress = { model: modelId, file: null, pct: 0 }; log(`[stt] downloading model ${modelId} (one-time, from Hugging Face)…`); }
|
|
197
|
+
try {
|
|
198
|
+
const rec = await parakeet.loadRecognizer({
|
|
199
|
+
modelId, dtype, allowDownload, log,
|
|
200
|
+
onProgress: (p) => { _progress = { model: modelId, file: p.file || null, pct: typeof p.pct === 'number' ? p.pct : (_progress?.pct ?? 0) }; },
|
|
201
|
+
});
|
|
202
|
+
// whisper-shaped adapter: ignores the whisper `{ language, task }` opts (parakeet
|
|
203
|
+
// auto-detects language) and returns { text }. `__parakeet` flags the language-ID
|
|
204
|
+
// short-circuit; `dispose` frees the ORT sessions on switch.
|
|
205
|
+
const adapter = async (audio) => ({ text: await rec.transcribe(audio) });
|
|
206
|
+
adapter.__parakeet = true;
|
|
207
|
+
adapter.dispose = () => rec.dispose();
|
|
208
|
+
_pipe = adapter; _model = modelId; _state = 'ready'; _err = null; _progress = null; _dtype = dtype;
|
|
209
|
+
if (prevPipe && prevPipe !== adapter) { try { await prevPipe.dispose?.(); } catch { /* ignore */ } }
|
|
210
|
+
log(`[stt] ready — model ${modelId} @ ${dtype} (parakeet-tdt, in-process, offline) — local dictation active`);
|
|
211
|
+
return true;
|
|
212
|
+
} catch (e) {
|
|
213
|
+
_err = e.message; _progress = null;
|
|
214
|
+
if (prevPipe) { _pipe = prevPipe; _model = prevModel; _state = 'ready'; }
|
|
215
|
+
else { _state = 'error'; }
|
|
216
|
+
log(`[stt] model load failed (${e.message})${prevPipe ? ' — keeping previous model' : ''}`);
|
|
217
|
+
return false;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// Parakeet only ships int8 + fp32 exports (no whisper-style q8). int8 loads and runs on
|
|
222
|
+
// BOTH runtimes; only force fp32 if a caller explicitly asks. Independent of the whisper
|
|
223
|
+
// runtimeDtype (which returns q8/fp32).
|
|
224
|
+
function PARAKEET_RUNTIME_DTYPE() { return parakeet.PARAKEET_DEFAULT_DTYPE; }
|
|
225
|
+
|
|
163
226
|
// Load the configured model once, on FIRST USE (never at gateway startup — the
|
|
164
227
|
// download is deferred until someone actually dictates). Single-flight.
|
|
165
228
|
export function init(cfg = {}) {
|
|
@@ -174,8 +237,11 @@ export function init(cfg = {}) {
|
|
|
174
237
|
export async function setModel(modelId, opts = {}) {
|
|
175
238
|
const log = typeof opts.onLog === 'function' ? opts.onLog : () => {};
|
|
176
239
|
if (!modelId) return false;
|
|
177
|
-
// Re-load if the model OR the requested precision changed.
|
|
178
|
-
|
|
240
|
+
// Re-load if the model OR the requested precision changed. Parakeet has its own
|
|
241
|
+
// dtype domain (int8/fp32), independent of whisper's q8/fp32 runtimeDtype.
|
|
242
|
+
const wantDtype = sttModelEngine(modelId) === 'parakeet-tdt'
|
|
243
|
+
? parakeet.parakeetDtype(opts.dtype && opts.dtype !== 'auto' ? opts.dtype : parakeet.PARAKEET_DEFAULT_DTYPE)
|
|
244
|
+
: (opts.dtype && opts.dtype !== 'auto' ? opts.dtype : (sttModelDtype(modelId) || runtimeDtype()));
|
|
179
245
|
if (modelId === _model && isReady() && _dtype === wantDtype) return true;
|
|
180
246
|
return loadModel(modelId, { log, allowDownload: opts.allowDownload !== false, dtype: opts.dtype });
|
|
181
247
|
}
|
|
@@ -348,7 +414,7 @@ async function decodeSession(s, { flush = false } = {}) {
|
|
|
348
414
|
|
|
349
415
|
// Auto-detect the spoken language on the session's first voiced audio (≥1s),
|
|
350
416
|
// then pin it. An explicit client `lang` wins; `.en` models skip all of this.
|
|
351
|
-
if (!s.lang && !s.langTried && !isEnglishOnly(_model) && audio.length >= SAMPLE_RATE) {
|
|
417
|
+
if (!s.lang && !s.langTried && !isEnglishOnly(_model) && !_pipe?.__parakeet && audio.length >= SAMPLE_RATE) {
|
|
352
418
|
s.langTried = true;
|
|
353
419
|
const detected = await detectLanguage(audio);
|
|
354
420
|
if (detected) { s.lang = detected; emit(s, { type: 'language', lang: detected }); }
|
package/src/stt-models.js
CHANGED
|
@@ -53,8 +53,35 @@ export const STT_MODEL_CATALOG = [
|
|
|
53
53
|
ramMB: 3200,
|
|
54
54
|
note: 'Best accuracy. For powerful machines; use the native (npm) gateway for speed.',
|
|
55
55
|
},
|
|
56
|
+
{
|
|
57
|
+
// NOT a whisper/seq2seq model — a NeMo TDT transducer. It doesn't run through the
|
|
58
|
+
// transformers.js ASR pipeline; the STT engine routes `engine: 'parakeet-tdt'`
|
|
59
|
+
// models to parakeet-engine.js (raw onnxruntime + a greedy TDT decode). Multilingual
|
|
60
|
+
// (25 European languages, auto-detected) and ~35× realtime at int8 on the native
|
|
61
|
+
// (npm) gateway — the fast local-dictation path. WASM runs it but slower.
|
|
62
|
+
id: 'istupakov/parakeet-tdt-0.6b-v3-onnx',
|
|
63
|
+
label: 'Parakeet TDT 0.6B v3 (multilingual, fast)',
|
|
64
|
+
lang: '25 European languages (auto-detected)',
|
|
65
|
+
tier: 'accurate',
|
|
66
|
+
engine: 'parakeet-tdt',
|
|
67
|
+
recommended: true, // our default recommendation: faster + more accurate than Whisper.
|
|
68
|
+
approxMB: 690, // int8: encoder 652 + decoder_joint 18 + preprocessor
|
|
69
|
+
ramMB: 1600,
|
|
70
|
+
note: 'Recommended — NVIDIA Parakeet transducer. Several× faster than Whisper at similar or better accuracy, English + 24 EU languages. One-time download; best on the native (npm) gateway.',
|
|
71
|
+
},
|
|
56
72
|
];
|
|
57
73
|
|
|
74
|
+
// The model we steer users to (a bigger, on-demand download — NOT the boot default,
|
|
75
|
+
// which stays a small model so first dictation works instantly). The settings UI
|
|
76
|
+
// surfaces this so users can install it after the gateway is running.
|
|
77
|
+
export const RECOMMENDED_STT_MODEL = 'istupakov/parakeet-tdt-0.6b-v3-onnx';
|
|
78
|
+
|
|
79
|
+
// STT engine backing a model. Default 'whisper' = the transformers.js ASR pipeline;
|
|
80
|
+
// 'parakeet-tdt' = the raw-onnxruntime transducer engine (parakeet-engine.js).
|
|
81
|
+
export function sttModelEngine(id) {
|
|
82
|
+
return sttModel(id)?.engine || 'whisper';
|
|
83
|
+
}
|
|
84
|
+
|
|
58
85
|
export function isKnownSttModel(id) {
|
|
59
86
|
return STT_MODEL_CATALOG.some((m) => m.id === id);
|
|
60
87
|
}
|