@openparachute/vault 0.7.4 → 0.7.5-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/mirror-import-jobs.ts +197 -0
- package/src/mirror-import.test.ts +102 -2
- package/src/mirror-import.ts +182 -22
- package/src/mirror-routes.test.ts +222 -3
- package/src/mirror-routes.ts +228 -86
- package/src/routing.ts +22 -6
- package/src/server.ts +62 -1
- package/src/transcription/models.test.ts +87 -0
- package/src/transcription/models.ts +187 -0
- package/src/transcription/providers/whisper-cpp.test.ts +218 -0
- package/src/transcription/providers/whisper-cpp.ts +241 -0
- package/src/transcription/resolve-binary.test.ts +131 -0
- package/src/transcription/resolve-binary.ts +111 -0
- package/src/transcription/select.ts +26 -3
- package/web/ui/dist/assets/index-CD4kPSY9.js +61 -0
- package/web/ui/dist/index.html +1 -1
- package/web/ui/dist/assets/index-NvwxfZcu.js +0 -61
package/src/mirror-routes.ts
CHANGED
|
@@ -76,8 +76,17 @@ import {
|
|
|
76
76
|
cloneAndImport,
|
|
77
77
|
type GitSpawn,
|
|
78
78
|
type ImportAuth,
|
|
79
|
+
type ImportProgress,
|
|
79
80
|
type ImportResult,
|
|
80
81
|
} from "./mirror-import.ts";
|
|
82
|
+
import {
|
|
83
|
+
ImportJobConflictError,
|
|
84
|
+
getImportJob,
|
|
85
|
+
getRunningImportJob,
|
|
86
|
+
startImportJob,
|
|
87
|
+
type ImportJob,
|
|
88
|
+
type ImportJobError,
|
|
89
|
+
} from "./mirror-import-jobs.ts";
|
|
81
90
|
import { redactToken } from "./export-watch.ts";
|
|
82
91
|
import {
|
|
83
92
|
findConflictingVault,
|
|
@@ -1668,12 +1677,24 @@ export async function applyCredentialsToMirror(
|
|
|
1668
1677
|
// - Sync setup throws → import result still returned (success);
|
|
1669
1678
|
// `sync_enabled: false` + a warning. Import success is never lost.
|
|
1670
1679
|
//
|
|
1671
|
-
// Response:
|
|
1672
|
-
//
|
|
1673
|
-
//
|
|
1674
|
-
// 400 { error, error_type, message } — validation
|
|
1675
|
-
// 409 { error, error_type, message }
|
|
1676
|
-
//
|
|
1680
|
+
// Response (vault#640 — ASYNC by default):
|
|
1681
|
+
// 202 { job_id, vault_name, status, stage, started_at, … }
|
|
1682
|
+
// — import started; poll the status route
|
|
1683
|
+
// 400 { error, error_type, message } — validation
|
|
1684
|
+
// 409 { error, error_type, message, job_id? }
|
|
1685
|
+
// — an import is already running here
|
|
1686
|
+
// 503 { error, error_type, message } — git isn't installed on this box
|
|
1687
|
+
//
|
|
1688
|
+
// Terminal outcomes (not-a-vault-export, clone failure, import counts) arrive
|
|
1689
|
+
// on the JOB record, not this response:
|
|
1690
|
+
// GET /vault/<name>/.parachute/mirror/import/<job_id>
|
|
1691
|
+
// 200 { status: "running", stage, detail? }
|
|
1692
|
+
// 200 { status: "succeeded", result: { notes_imported, … } }
|
|
1693
|
+
// 200 { status: "failed", error: { error_type, message } }
|
|
1694
|
+
// 404 { error_type: "job_not_found" }
|
|
1695
|
+
//
|
|
1696
|
+
// `{ "wait": true }` in the POST body restores the old synchronous 200 shape
|
|
1697
|
+
// for scripted callers. See the note in the handler.
|
|
1677
1698
|
//
|
|
1678
1699
|
// Admin-gated upstream in routing.ts.
|
|
1679
1700
|
// ---------------------------------------------------------------------------
|
|
@@ -1681,10 +1702,13 @@ export async function applyCredentialsToMirror(
|
|
|
1681
1702
|
/**
|
|
1682
1703
|
* `POST /vault/<name>/.parachute/mirror/import`. See block comment above.
|
|
1683
1704
|
*
|
|
1684
|
-
*
|
|
1685
|
-
*
|
|
1686
|
-
*
|
|
1687
|
-
*
|
|
1705
|
+
* Starts a background job and returns 202 immediately. Before vault#640 this
|
|
1706
|
+
* ran the whole clone-and-import inside the request; that could not work for a
|
|
1707
|
+
* real vault, because hub fronts this route with
|
|
1708
|
+
* `Bun.serve({ idleTimeout: 255 })` and kills anything past ~4 minutes
|
|
1709
|
+
* regardless of what vault does. The 60s clone cap the old code carried was a
|
|
1710
|
+
* symptom of the same constraint, and it made every non-trivial vault
|
|
1711
|
+
* un-importable with "git clone timed out after 60s".
|
|
1688
1712
|
*
|
|
1689
1713
|
* `spawnOverride` is a test seam: lets the test inject a fake git binary.
|
|
1690
1714
|
* Production callers omit it; `cloneAndImport` falls back to `defaultGitSpawn`.
|
|
@@ -1712,6 +1736,7 @@ export async function handleMirrorImport(
|
|
|
1712
1736
|
credentials?: unknown;
|
|
1713
1737
|
enable_sync?: unknown;
|
|
1714
1738
|
override?: unknown;
|
|
1739
|
+
wait?: unknown;
|
|
1715
1740
|
};
|
|
1716
1741
|
try {
|
|
1717
1742
|
body = (await req.json()) as Record<string, unknown>;
|
|
@@ -1804,10 +1829,22 @@ export async function handleMirrorImport(
|
|
|
1804
1829
|
);
|
|
1805
1830
|
}
|
|
1806
1831
|
|
|
1807
|
-
//
|
|
1808
|
-
//
|
|
1809
|
-
//
|
|
1810
|
-
|
|
1832
|
+
// `enable_sync` defaults FALSE when omitted (vault#641).
|
|
1833
|
+
//
|
|
1834
|
+
// vault#416 shipped this default-ON: importing a repo silently turned that
|
|
1835
|
+
// repo into this vault's push target. That inverts the risk. Import is a
|
|
1836
|
+
// READ — the operator is pulling a vault onto a new box — and the natural
|
|
1837
|
+
// reading of "import from this repo" does not include "and start writing
|
|
1838
|
+
// back to it." Default-on meant the safe intent (pull a copy) required
|
|
1839
|
+
// noticing and unchecking a box, while the destructive one (rewire backup,
|
|
1840
|
+
// potentially clobbering the repo a DIFFERENT box is backing up to) was what
|
|
1841
|
+
// you got by not reading carefully. Worse, it made the credential question
|
|
1842
|
+
// incoherent: operators reasonably refused to supply a token to *read* a
|
|
1843
|
+
// private repo because supplying it appeared to arm a write.
|
|
1844
|
+
//
|
|
1845
|
+
// Opting in is now explicit. Only a literal `true` enables it; any
|
|
1846
|
+
// non-boolean is a validation error so a malformed body can't flip it.
|
|
1847
|
+
let enableSync = false;
|
|
1811
1848
|
if ("enable_sync" in body && body.enable_sync !== undefined) {
|
|
1812
1849
|
if (typeof body.enable_sync !== "boolean") {
|
|
1813
1850
|
return Response.json(
|
|
@@ -1815,7 +1852,7 @@ export async function handleMirrorImport(
|
|
|
1815
1852
|
error: "enable_sync invalid",
|
|
1816
1853
|
error_type: "validation",
|
|
1817
1854
|
field: "enable_sync",
|
|
1818
|
-
message: "enable_sync must be a boolean (defaults to
|
|
1855
|
+
message: "enable_sync must be a boolean (defaults to false when omitted).",
|
|
1819
1856
|
},
|
|
1820
1857
|
{ status: 400 },
|
|
1821
1858
|
);
|
|
@@ -1823,6 +1860,14 @@ export async function handleMirrorImport(
|
|
|
1823
1860
|
enableSync = body.enable_sync;
|
|
1824
1861
|
}
|
|
1825
1862
|
|
|
1863
|
+
// Back-compat seam: `wait: true` restores the pre-vault#640 synchronous
|
|
1864
|
+
// response (200 + the ImportResult body) for scripted callers that read the
|
|
1865
|
+
// result straight off the POST. It inherits the new stall-based timeout, so
|
|
1866
|
+
// it is no longer capped at 60s — but it is still subject to whatever
|
|
1867
|
+
// request timeout sits between the caller and vault, which is exactly why it
|
|
1868
|
+
// is no longer the default. The SPA never sets it.
|
|
1869
|
+
const wait = body.wait === true;
|
|
1870
|
+
|
|
1826
1871
|
// vault#482: cross-vault clobber override for the sync-enable step. Default
|
|
1827
1872
|
// off; a literal `true` lets the operator deliberately point this vault's
|
|
1828
1873
|
// sync at a repo a SIBLING vault already backs up to (e.g. they moved the
|
|
@@ -1837,22 +1882,13 @@ export async function handleMirrorImport(
|
|
|
1837
1882
|
const store = getVaultStore(vaultName);
|
|
1838
1883
|
const assets = assetsDir(vaultName);
|
|
1839
1884
|
|
|
1840
|
-
|
|
1885
|
+
// Preflight git BEFORE the job starts. `cloneAndImport` preflights too, but
|
|
1886
|
+
// doing it here keeps "git isn't installed" a synchronous, actionable 503
|
|
1887
|
+
// instead of a job the operator has to poll to discover was doomed.
|
|
1841
1888
|
try {
|
|
1842
|
-
|
|
1843
|
-
vaultName,
|
|
1844
|
-
remoteUrl: remote_url,
|
|
1845
|
-
auth,
|
|
1846
|
-
mode,
|
|
1847
|
-
store,
|
|
1848
|
-
assetsDir: assets,
|
|
1849
|
-
spawn: spawnOverride,
|
|
1850
|
-
which: whichOverride,
|
|
1851
|
-
});
|
|
1889
|
+
ensureGitAvailable(whichOverride);
|
|
1852
1890
|
} catch (err) {
|
|
1853
1891
|
if (err instanceof GitNotInstalledError) {
|
|
1854
|
-
// 503 Service Unavailable — the server isn't configured to do this
|
|
1855
|
-
// yet (git missing). The message tells the operator how to fix it.
|
|
1856
1892
|
return Response.json(
|
|
1857
1893
|
{
|
|
1858
1894
|
error: "git not installed",
|
|
@@ -1862,82 +1898,188 @@ export async function handleMirrorImport(
|
|
|
1862
1898
|
{ status: 503 },
|
|
1863
1899
|
);
|
|
1864
1900
|
}
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1901
|
+
throw err;
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
/**
|
|
1905
|
+
* The actual work, shared by the async (default) and `wait: true` paths.
|
|
1906
|
+
* Runs the clone+import, then optionally wires push-back. Every failure
|
|
1907
|
+
* after a SUCCESSFUL import is non-fatal — an import is never lost to a
|
|
1908
|
+
* sync-setup error.
|
|
1909
|
+
*/
|
|
1910
|
+
const runImport = async (
|
|
1911
|
+
onProgress: (update: ImportProgress) => void,
|
|
1912
|
+
): Promise<ImportResult> => {
|
|
1913
|
+
const result = await cloneAndImport({
|
|
1914
|
+
vaultName,
|
|
1915
|
+
remoteUrl: remote_url,
|
|
1916
|
+
auth,
|
|
1917
|
+
mode,
|
|
1918
|
+
store,
|
|
1919
|
+
assetsDir: assets,
|
|
1920
|
+
spawn: spawnOverride,
|
|
1921
|
+
which: whichOverride,
|
|
1922
|
+
onProgress,
|
|
1923
|
+
});
|
|
1924
|
+
|
|
1925
|
+
if (enableSync) {
|
|
1926
|
+
onProgress({ stage: "syncing" });
|
|
1927
|
+
try {
|
|
1928
|
+
const manager = managerOverride ?? getMirrorManager(vaultName) ?? undefined;
|
|
1929
|
+
const outcome = await enableSyncToImportedRepo({
|
|
1930
|
+
vaultName,
|
|
1931
|
+
remoteUrl: remote_url,
|
|
1932
|
+
auth,
|
|
1933
|
+
manager,
|
|
1934
|
+
override,
|
|
1935
|
+
});
|
|
1936
|
+
result.sync_enabled = outcome.sync_enabled;
|
|
1937
|
+
if (outcome.warning) result.sync_warning = outcome.warning;
|
|
1938
|
+
} catch (err) {
|
|
1939
|
+
// Defense-in-depth: enableSyncToImportedRepo is written to not throw
|
|
1940
|
+
// (it catches its own write/credential/reload errors), but a future
|
|
1941
|
+
// edit or an unexpected throw must NOT take down a successful import.
|
|
1942
|
+
const msg = redactToken((err as Error).message ?? String(err));
|
|
1943
|
+
console.warn(
|
|
1944
|
+
`[mirror-import] sync-enable threw after a successful import (non-fatal): ${msg}`,
|
|
1945
|
+
);
|
|
1946
|
+
result.sync_enabled = false;
|
|
1947
|
+
result.sync_warning =
|
|
1948
|
+
"Import succeeded, but enabling Sync failed. Set up Sync separately from the Git remote section.";
|
|
1949
|
+
}
|
|
1874
1950
|
}
|
|
1875
|
-
|
|
1951
|
+
return result;
|
|
1952
|
+
};
|
|
1953
|
+
|
|
1954
|
+
// ---- Back-compat synchronous path (`wait: true`) ------------------------
|
|
1955
|
+
if (wait) {
|
|
1956
|
+
try {
|
|
1957
|
+
const result = await runImport(() => {});
|
|
1958
|
+
return Response.json(result, {
|
|
1959
|
+
headers: { "Access-Control-Allow-Origin": "*" },
|
|
1960
|
+
});
|
|
1961
|
+
} catch (err) {
|
|
1962
|
+
const { error_type, message } = classifyImportError(err);
|
|
1876
1963
|
return Response.json(
|
|
1877
|
-
{
|
|
1878
|
-
|
|
1879
|
-
error_type: "not_a_vault_export",
|
|
1880
|
-
message: err.message,
|
|
1881
|
-
},
|
|
1882
|
-
{ status: 400 },
|
|
1964
|
+
{ error: importErrorTitle(error_type), error_type, message },
|
|
1965
|
+
{ status: importErrorStatus(error_type) },
|
|
1883
1966
|
);
|
|
1884
1967
|
}
|
|
1885
|
-
|
|
1968
|
+
}
|
|
1969
|
+
|
|
1970
|
+
// ---- Async path (default) ----------------------------------------------
|
|
1971
|
+
let job: ImportJob;
|
|
1972
|
+
try {
|
|
1973
|
+
job = startImportJob(vaultName, runImport, classifyImportError);
|
|
1974
|
+
} catch (err) {
|
|
1975
|
+
if (err instanceof ImportJobConflictError) {
|
|
1976
|
+
const existing = getRunningImportJob(vaultName);
|
|
1886
1977
|
return Response.json(
|
|
1887
1978
|
{
|
|
1888
|
-
error: "
|
|
1889
|
-
error_type: "
|
|
1979
|
+
error: "Import already running",
|
|
1980
|
+
error_type: "concurrent_import",
|
|
1890
1981
|
message: err.message,
|
|
1982
|
+
// Hand back the in-flight job so a second tab can attach to the
|
|
1983
|
+
// running import's progress rather than just being told "no".
|
|
1984
|
+
job_id: existing?.job_id,
|
|
1891
1985
|
},
|
|
1892
|
-
{ status:
|
|
1986
|
+
{ status: 409 },
|
|
1893
1987
|
);
|
|
1894
1988
|
}
|
|
1989
|
+
throw err;
|
|
1990
|
+
}
|
|
1991
|
+
|
|
1992
|
+
return Response.json(job, {
|
|
1993
|
+
status: 202,
|
|
1994
|
+
headers: { "Access-Control-Allow-Origin": "*" },
|
|
1995
|
+
});
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
/**
|
|
1999
|
+
* `GET /vault/<name>/.parachute/mirror/import/<job_id>` — poll an import.
|
|
2000
|
+
*
|
|
2001
|
+
* Returns the job record (status / stage / detail, plus `result` or `error`
|
|
2002
|
+
* once terminal). 404 when the id is unknown for this vault — which is also
|
|
2003
|
+
* what a caller sees after a vault restart, since jobs are in-memory by design
|
|
2004
|
+
* (see `mirror-import-jobs.ts`).
|
|
2005
|
+
*
|
|
2006
|
+
* Admin-gated upstream in routing.ts, same as the POST.
|
|
2007
|
+
*/
|
|
2008
|
+
export function handleMirrorImportStatus(
|
|
2009
|
+
vaultName: string,
|
|
2010
|
+
jobId: string,
|
|
2011
|
+
): Response {
|
|
2012
|
+
const job = getImportJob(vaultName, jobId);
|
|
2013
|
+
if (!job) {
|
|
1895
2014
|
return Response.json(
|
|
1896
2015
|
{
|
|
1897
|
-
error: "
|
|
1898
|
-
error_type: "
|
|
1899
|
-
message:
|
|
2016
|
+
error: "No such import job",
|
|
2017
|
+
error_type: "job_not_found",
|
|
2018
|
+
message:
|
|
2019
|
+
"That import job isn't known to this vault. It may have finished more than an hour ago, or the vault restarted while it was running.",
|
|
1900
2020
|
},
|
|
1901
|
-
{ status:
|
|
2021
|
+
{ status: 404, headers: { "Access-Control-Allow-Origin": "*" } },
|
|
1902
2022
|
);
|
|
1903
2023
|
}
|
|
2024
|
+
return Response.json(job, {
|
|
2025
|
+
headers: {
|
|
2026
|
+
"Access-Control-Allow-Origin": "*",
|
|
2027
|
+
// Polled endpoint — never let an intermediary serve a stale stage.
|
|
2028
|
+
"cache-control": "no-store",
|
|
2029
|
+
},
|
|
2030
|
+
});
|
|
2031
|
+
}
|
|
1904
2032
|
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
// carries `sync_enabled: false` (set by importResultFromStats); we flip it
|
|
1910
|
-
// true only when sync is actually wired (or already wired to this remote).
|
|
1911
|
-
if (enableSync) {
|
|
1912
|
-
try {
|
|
1913
|
-
const manager =
|
|
1914
|
-
managerOverride ?? getMirrorManager(vaultName) ?? undefined;
|
|
1915
|
-
const outcome = await enableSyncToImportedRepo({
|
|
1916
|
-
vaultName,
|
|
1917
|
-
remoteUrl: remote_url,
|
|
1918
|
-
auth,
|
|
1919
|
-
manager,
|
|
1920
|
-
override,
|
|
1921
|
-
});
|
|
1922
|
-
result.sync_enabled = outcome.sync_enabled;
|
|
1923
|
-
if (outcome.warning) result.sync_warning = outcome.warning;
|
|
1924
|
-
} catch (err) {
|
|
1925
|
-
// Defense-in-depth: enableSyncToImportedRepo is written to not throw
|
|
1926
|
-
// (it catches its own write/credential/reload errors), but a future
|
|
1927
|
-
// edit or an unexpected throw must NOT take down a successful import.
|
|
1928
|
-
const msg = redactToken((err as Error).message ?? String(err));
|
|
1929
|
-
console.warn(
|
|
1930
|
-
`[mirror-import] sync-enable threw after a successful import (non-fatal): ${msg}`,
|
|
1931
|
-
);
|
|
1932
|
-
result.sync_enabled = false;
|
|
1933
|
-
result.sync_warning =
|
|
1934
|
-
"Import succeeded, but enabling Sync failed. Set up Sync separately from the Git remote section.";
|
|
1935
|
-
}
|
|
2033
|
+
/** Map a thrown import error onto the wire's `error_type` vocabulary. */
|
|
2034
|
+
function classifyImportError(err: unknown): ImportJobError {
|
|
2035
|
+
if (err instanceof GitNotInstalledError) {
|
|
2036
|
+
return { error_type: "git_not_installed", message: err.message };
|
|
1936
2037
|
}
|
|
2038
|
+
if (err instanceof ImportConflictError) {
|
|
2039
|
+
return { error_type: "concurrent_import", message: err.message };
|
|
2040
|
+
}
|
|
2041
|
+
if (err instanceof NotAVaultExportError) {
|
|
2042
|
+
return { error_type: "not_a_vault_export", message: err.message };
|
|
2043
|
+
}
|
|
2044
|
+
if (err instanceof CloneFailedError) {
|
|
2045
|
+
return { error_type: "clone_failed", message: err.message };
|
|
2046
|
+
}
|
|
2047
|
+
return {
|
|
2048
|
+
error_type: "internal",
|
|
2049
|
+
message: redactToken((err as Error)?.message ?? String(err)),
|
|
2050
|
+
};
|
|
2051
|
+
}
|
|
1937
2052
|
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
2053
|
+
/** HTTP status for an `error_type` — used by the `wait: true` path. */
|
|
2054
|
+
function importErrorStatus(errorType: ImportJobError["error_type"]): number {
|
|
2055
|
+
switch (errorType) {
|
|
2056
|
+
case "git_not_installed":
|
|
2057
|
+
return 503;
|
|
2058
|
+
case "concurrent_import":
|
|
2059
|
+
return 409;
|
|
2060
|
+
case "not_a_vault_export":
|
|
2061
|
+
return 400;
|
|
2062
|
+
case "clone_failed":
|
|
2063
|
+
return 502;
|
|
2064
|
+
default:
|
|
2065
|
+
return 500;
|
|
2066
|
+
}
|
|
2067
|
+
}
|
|
2068
|
+
|
|
2069
|
+
/** Human title for an `error_type` — used by the `wait: true` path. */
|
|
2070
|
+
function importErrorTitle(errorType: ImportJobError["error_type"]): string {
|
|
2071
|
+
switch (errorType) {
|
|
2072
|
+
case "git_not_installed":
|
|
2073
|
+
return "git not installed";
|
|
2074
|
+
case "concurrent_import":
|
|
2075
|
+
return "Import already running";
|
|
2076
|
+
case "not_a_vault_export":
|
|
2077
|
+
return "Not a vault export";
|
|
2078
|
+
case "clone_failed":
|
|
2079
|
+
return "Clone failed";
|
|
2080
|
+
default:
|
|
2081
|
+
return "Import failed";
|
|
2082
|
+
}
|
|
1941
2083
|
}
|
|
1942
2084
|
|
|
1943
2085
|
// ---------------------------------------------------------------------------
|
package/src/routing.ts
CHANGED
|
@@ -112,6 +112,7 @@ import {
|
|
|
112
112
|
handleMirrorHistory,
|
|
113
113
|
handleMirrorHistoryShow,
|
|
114
114
|
handleMirrorImport,
|
|
115
|
+
handleMirrorImportStatus,
|
|
115
116
|
handleMirrorPushNow,
|
|
116
117
|
handleMirrorPut,
|
|
117
118
|
handleMirrorRunNow,
|
|
@@ -874,12 +875,18 @@ export async function route(
|
|
|
874
875
|
return handleMirrorHistory(req, manager);
|
|
875
876
|
}
|
|
876
877
|
|
|
877
|
-
// /.parachute/mirror/import
|
|
878
|
-
//
|
|
879
|
-
//
|
|
878
|
+
// /.parachute/mirror/import — POST: start a clone+import job (202).
|
|
879
|
+
// /.parachute/mirror/import/<id> — GET: poll that job.
|
|
880
|
+
//
|
|
881
|
+
// Admin-gated. Async since vault#640 — the synchronous shape couldn't
|
|
882
|
+
// survive hub's 255s proxy idleTimeout, which is what capped imports at a
|
|
883
|
+
// vault-sized demo. See mirror-routes.ts:handleMirrorImport for the
|
|
880
884
|
// request/response shape + error map. Symmetric counterpart to the
|
|
881
885
|
// export-to-git flow vault#382 + vault#384 shipped.
|
|
882
|
-
if (
|
|
886
|
+
if (
|
|
887
|
+
subpath === "/.parachute/mirror/import" ||
|
|
888
|
+
subpath.startsWith("/.parachute/mirror/import/")
|
|
889
|
+
) {
|
|
883
890
|
if (!hasScopeForVault(auth.scopes, vaultName, "admin")) {
|
|
884
891
|
return Response.json(
|
|
885
892
|
{
|
|
@@ -892,10 +899,19 @@ export async function route(
|
|
|
892
899
|
{ status: 403 },
|
|
893
900
|
);
|
|
894
901
|
}
|
|
895
|
-
if (
|
|
902
|
+
if (subpath === "/.parachute/mirror/import") {
|
|
903
|
+
if (req.method !== "POST") {
|
|
904
|
+
return Response.json({ error: "Method not allowed" }, { status: 405 });
|
|
905
|
+
}
|
|
906
|
+
return handleMirrorImport(req, vaultName);
|
|
907
|
+
}
|
|
908
|
+
if (req.method !== "GET") {
|
|
896
909
|
return Response.json({ error: "Method not allowed" }, { status: 405 });
|
|
897
910
|
}
|
|
898
|
-
|
|
911
|
+
const jobId = decodeURIComponent(
|
|
912
|
+
subpath.slice("/.parachute/mirror/import/".length),
|
|
913
|
+
);
|
|
914
|
+
return handleMirrorImportStatus(vaultName, jobId);
|
|
899
915
|
}
|
|
900
916
|
|
|
901
917
|
// /.parachute/mirror/auth/* — UI-configurable git push credentials.
|
package/src/server.ts
CHANGED
|
@@ -36,6 +36,7 @@ import { TranscribeCppProvider } from "./transcription/providers/transcribe-cpp.
|
|
|
36
36
|
import { ParakeetMlxProvider } from "./transcription/providers/parakeet-mlx.ts";
|
|
37
37
|
import { OnnxAsrProvider } from "./transcription/providers/onnx-asr.ts";
|
|
38
38
|
import {
|
|
39
|
+
resolveTranscriptionModelId,
|
|
39
40
|
resolveTranscriptionProviderName,
|
|
40
41
|
resolveTranscribeCppPaths,
|
|
41
42
|
transcribeCppInstalled,
|
|
@@ -63,6 +64,15 @@ import {
|
|
|
63
64
|
} from "./mirror-config.ts";
|
|
64
65
|
import { GLOBAL_CONFIG_PATH } from "./config.ts";
|
|
65
66
|
import { selfRegister } from "./self-register.ts";
|
|
67
|
+
import { join } from "node:path";
|
|
68
|
+
import { TRANSCRIPTION_MODELS, findModel } from "./transcription/models.ts";
|
|
69
|
+
import {
|
|
70
|
+
candidateBinDirs,
|
|
71
|
+
managedModelDir,
|
|
72
|
+
resolveCliBinary,
|
|
73
|
+
resolveFfmpeg,
|
|
74
|
+
} from "./transcription/resolve-binary.ts";
|
|
75
|
+
import { WhisperCppProvider } from "./transcription/providers/whisper-cpp.ts";
|
|
66
76
|
import { createSubscribeWsBinding, isWebSocketUpgrade } from "./ws-server.ts";
|
|
67
77
|
import { warnLegacyGlobalApiKeys } from "./auth.ts";
|
|
68
78
|
import pkg from "../package.json" with { type: "json" };
|
|
@@ -149,7 +159,58 @@ const commonWorkerOpts = {
|
|
|
149
159
|
// Provider selection (scribe-fold Phase 2a). Default is `scribe-http` — unset
|
|
150
160
|
// TRANSCRIPTION_PROVIDER means the existing scribe-http path runs unchanged.
|
|
151
161
|
const providerName = resolveTranscriptionProviderName();
|
|
152
|
-
if (providerName === "
|
|
162
|
+
if (providerName === "whisper-cpp") {
|
|
163
|
+
// whisper.cpp's prebuilt CLIs — the local, no-Python path that actually
|
|
164
|
+
// ships binaries on both macOS (brew bottle) and Linux (release tarball).
|
|
165
|
+
// The model decides which CLI runs: parakeet-cli or whisper-cli.
|
|
166
|
+
//
|
|
167
|
+
// Binaries are resolved through the ladder in `resolve-binary.ts` rather
|
|
168
|
+
// than PATH alone, because a launchd-supervised vault does not inherit a
|
|
169
|
+
// login shell's PATH — on a Mac, `brew install whisper-cpp` otherwise looks
|
|
170
|
+
// installed to the operator and invisible to us.
|
|
171
|
+
const model = findModel(resolveTranscriptionModelId());
|
|
172
|
+
if (!model) {
|
|
173
|
+
console.warn(
|
|
174
|
+
`[transcribe] TRANSCRIPTION_MODEL="${resolveTranscriptionModelId()}" is not a known model — ` +
|
|
175
|
+
`valid ids: ${TRANSCRIPTION_MODELS.map((m) => m.id).join(", ")}`,
|
|
176
|
+
);
|
|
177
|
+
} else {
|
|
178
|
+
const binPath = resolveCliBinary(model.engine);
|
|
179
|
+
const modelPath = join(managedModelDir(), model.filename);
|
|
180
|
+
const ffmpegPath = resolveFfmpeg();
|
|
181
|
+
if (binPath && existsSync(modelPath)) {
|
|
182
|
+
transcriptionWorker = startTranscriptionWorker({
|
|
183
|
+
...commonWorkerOpts,
|
|
184
|
+
provider: new WhisperCppProvider({
|
|
185
|
+
binPath,
|
|
186
|
+
engine: model.engine,
|
|
187
|
+
modelPath,
|
|
188
|
+
ffmpegPath,
|
|
189
|
+
}),
|
|
190
|
+
});
|
|
191
|
+
wireTranscriptionWorker(transcriptionWorker);
|
|
192
|
+
console.log(`[transcribe] worker started → whisper-cpp (${model.label}, ${binPath})`);
|
|
193
|
+
if (!ffmpegPath) {
|
|
194
|
+
console.warn(
|
|
195
|
+
"[transcribe] ffmpeg was not found — audio still has to be transcoded to 16 kHz mono " +
|
|
196
|
+
"WAV, so transcription will fail until it's installed (`brew install ffmpeg` / `apt install ffmpeg`).",
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
} else {
|
|
200
|
+
// Say WHICH piece is missing and where we looked. "Not installed" with
|
|
201
|
+
// two possible meanings and different fixes for each is not actionable.
|
|
202
|
+
const missing = [
|
|
203
|
+
binPath ? null : `the ${model.engine === "whisper" ? "whisper-cli" : "parakeet-cli"} binary`,
|
|
204
|
+
existsSync(modelPath) ? null : `the model (${model.label}, ${model.sizeMb} MB)`,
|
|
205
|
+
].filter(Boolean);
|
|
206
|
+
console.warn(
|
|
207
|
+
`[transcribe] TRANSCRIPTION_PROVIDER=whisper-cpp but ${missing.join(" and ")} ` +
|
|
208
|
+
`${missing.length > 1 ? "are" : "is"} missing — run \`parachute-vault transcription install\`. ` +
|
|
209
|
+
`Searched for binaries in: ${candidateBinDirs().slice(0, 4).join(", ")}…`,
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
} else if (providerName === "transcribe-cpp") {
|
|
153
214
|
// Local, no-Python provider: subprocess a transcribe-cli. Only start the
|
|
154
215
|
// worker when a runnable CLI + model are actually present — otherwise every
|
|
155
216
|
// pending item would terminal-fail with `missing_provider`. (v0.1.1 ships a
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model catalog.
|
|
3
|
+
*
|
|
4
|
+
* These tests care about two things a catalog gets wrong quietly: URLs that
|
|
5
|
+
* don't match the filename we save to (so a re-run re-downloads forever), and
|
|
6
|
+
* a default-picker that hands a 1.5 GB model to a 1 GB box.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { describe, expect, test } from "bun:test";
|
|
10
|
+
import {
|
|
11
|
+
DEFAULT_MODEL_ID,
|
|
12
|
+
findModel,
|
|
13
|
+
pickDefaultModel,
|
|
14
|
+
TRANSCRIPTION_MODELS,
|
|
15
|
+
} from "./models.ts";
|
|
16
|
+
|
|
17
|
+
describe("catalog integrity", () => {
|
|
18
|
+
test("ids are unique", () => {
|
|
19
|
+
const ids = TRANSCRIPTION_MODELS.map((m) => m.id);
|
|
20
|
+
expect(ids.length).toBe(new Set(ids).size);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
test("every URL's basename matches the filename we write", () => {
|
|
24
|
+
// Drift here means the downloader saves to a name the resolver never
|
|
25
|
+
// looks for, so every boot re-downloads and nothing ever becomes ready.
|
|
26
|
+
for (const m of TRANSCRIPTION_MODELS) {
|
|
27
|
+
expect(m.url.split("/").pop()).toBe(m.filename);
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("Parakeet models come from ggml-org, Whisper from ggerganov", () => {
|
|
32
|
+
// Load-bearing: handy-computer's GGUFs are NOT loadable by whisper.cpp's
|
|
33
|
+
// parakeet-cli (verified — "failed to load Parakeet model"). Pointing a
|
|
34
|
+
// Parakeet entry at handy-computer would produce a model that downloads
|
|
35
|
+
// fine and then fails at every transcription.
|
|
36
|
+
for (const m of TRANSCRIPTION_MODELS) {
|
|
37
|
+
if (m.engine === "parakeet") expect(m.url).toContain("ggml-org/parakeet-GGUF");
|
|
38
|
+
else expect(m.url).toContain("ggerganov/whisper.cpp");
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("catalog is ordered smallest-first", () => {
|
|
43
|
+
const sizes = TRANSCRIPTION_MODELS.map((m) => m.sizeMb);
|
|
44
|
+
expect([...sizes].sort((a, b) => a - b)).toEqual(sizes);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test("the default exists and is a Parakeet model", () => {
|
|
48
|
+
const d = findModel(DEFAULT_MODEL_ID);
|
|
49
|
+
expect(d).toBeDefined();
|
|
50
|
+
expect(d!.engine).toBe("parakeet");
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("covers the small and mid size classes an operator asks for", () => {
|
|
54
|
+
const sizes = TRANSCRIPTION_MODELS.map((m) => m.sizeMb);
|
|
55
|
+
expect(sizes.some((s) => s < 150)).toBe(true); // ~100 MB class
|
|
56
|
+
expect(sizes.some((s) => s >= 300 && s <= 700)).toBe(true); // ~400–700 MB class
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
describe("pickDefaultModel", () => {
|
|
61
|
+
test("a comfortable box gets the recommended Parakeet", () => {
|
|
62
|
+
expect(pickDefaultModel(16384).id).toBe(DEFAULT_MODEL_ID);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("a small box steps DOWN rather than swapping itself to death", () => {
|
|
66
|
+
const picked = pickDefaultModel(1024);
|
|
67
|
+
expect(picked.minRamMb).toBeLessThanOrEqual(1024);
|
|
68
|
+
expect(picked.sizeMb).toBeLessThan(200);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test("never returns undefined, even on an absurdly small box", () => {
|
|
72
|
+
const picked = pickDefaultModel(64);
|
|
73
|
+
expect(picked).toBeDefined();
|
|
74
|
+
expect(picked.id).toBe(TRANSCRIPTION_MODELS[0]!.id);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test("a mid box gets something that fits its RAM floor", () => {
|
|
78
|
+
const picked = pickDefaultModel(2048);
|
|
79
|
+
expect(picked.minRamMb).toBeLessThanOrEqual(2048);
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
describe("findModel", () => {
|
|
84
|
+
test("unknown id → undefined, not a throw", () => {
|
|
85
|
+
expect(findModel("nope")).toBeUndefined();
|
|
86
|
+
});
|
|
87
|
+
});
|