@gjsify/model-store 0.64.25 → 0.64.26-next.312

package/README.md

@@ -51,6 +51,25 @@ Digests: a downloaded file is hashed while it streams and only renamed into plac
 
 The store never answers `unavailable` itself: that is the capability's call (no engine, or `checkFit` from `@gjsify/machine-fit` says `none`).
 
+## Downloading a file you did not plan for: `downloadVerified`
+
+Not every model comes from a fixed manifest. For a one-off download that still needs streaming verification and an atomic rename, use the standalone helper:
+
+```typescript
+import { downloadVerified } from '@gjsify/model-store';
+
+const { bytes, sha256 } = await downloadVerified({
+    url: 'https://example.org/one-off.bin',
+    dest: '/tmp/one-off.bin',
+    bytes: 2313101,
+    sha256: '6b99cbfd39246b6706f98ec13c7c50c6b299181f2474fa05cbc8046acc274396',
+    headers: { Authorization: 'Bearer token' },
+    monitor: (m) => m.addEventListener('downloadprogress', (e) => console.log(e.loaded)),
+});
+```
+
+`bytes` and `sha256` are optional checks, but the returned values always report what was actually received. The file is written under a temporary name and renamed into place only after the checks pass; failures remove the partial file.
+
 ## License
 
 MIT

package/lib/esm/download.js

@@ -0,0 +1 @@
+import"./_virtual/_rolldown/runtime.js";import{ProgressFanout as e}from"./progress.js";import{dirname as t}from"node:path";import{createHash as n,randomUUID as r}from"node:crypto";import{closeSync as i,mkdirSync as a,openSync as o,renameSync as s,unlinkSync as c,writeSync as l}from"node:fs";function removeQuietly(e){try{c(e)}catch{}}function prefix(e){return e===void 0?`Downloading`:`Downloading ${e}`}async function downloadVerifiedInternal(c){let{url:u,dest:d,bytes:f,sha256:p,headers:m,signal:h,fetch:g}=c,_=await(g??((e,t)=>fetch(e,t)))(u,{signal:h,redirect:`follow`,headers:m});if(!_.ok)throw Error(`${prefix(c.id)} from ${u} failed: HTTP ${_.status}`);let v=_.headers.get(`content-length`);if(v!==null&&f!==void 0&&Number(v)!==f){let e=c.id===void 0?`expected`:`the manifest says`;throw Error(`${prefix(c.id)} refused: ${u} declares ${v} bytes, ${e} ${f}`)}a(t(d),{recursive:!0});let y=`${d}.part-${r()}`,b=o(y,`w`),x=!1,S=c.progress??new e(c.id??u);c.progress===void 0&&c.monitor!==void 0&&S.add(c.monitor);try{let e=n(`sha256`),t=0,r=f,write=n=>{if(t+=n.length,f!==void 0&&t>f){let e=c.id===void 0?`the expected`:`the manifest's`;throw Error(`${prefix(c.id)} failed: more than ${e} ${f} bytes`)}e.update(n),l(b,n),S.report(t,r??t)};S.report(0,r??0);let a=_.body?.getReader();if(a!==void 0)try{for(;;){h?.throwIfAborted();let{done:e,value:t}=await a.read();if(e)break;t!==void 0&&t.length>0&&write(t)}}catch(e){throw await a.cancel().catch(()=>void 0),e}else write(new Uint8Array(await _.arrayBuffer()));if(f!==void 0&&t!==f)throw Error(`${prefix(c.id)} failed: got ${t} bytes, expected ${f}`);let o=e.digest(`hex`);if(p!==void 0&&o!==p)throw Error(`${prefix(c.id)} failed: sha256 ${o}, expected ${p}`);return i(b),x=!0,s(y,d),S.report(t,r??t),{bytes:t,sha256:o}}catch(e){throw x||i(b),removeQuietly(y),e}}async function downloadVerified(e){return downloadVerifiedInternal(e)}export{downloadVerified,downloadVerifiedInternal};

package/lib/esm/index.js

@@ -1 +1 @@
-import{modelsRoot as e,userCacheDir as t}from"./cache-dir.js";import{validateManifest as n}from"./manifest.js";import{DownloadProgressEvent as r}from"./progress.js";import{ModelStore as i,sha256File as a}from"./store.js";export{r as DownloadProgressEvent,i as ModelStore,e as modelsRoot,a as sha256File,t as userCacheDir,n as validateManifest};
+import{modelsRoot as e,userCacheDir as t}from"./cache-dir.js";import{DownloadProgressEvent as n}from"./progress.js";import{downloadVerified as r}from"./download.js";import{validateManifest as i}from"./manifest.js";import{ModelStore as a,sha256File as o}from"./store.js";export{n as DownloadProgressEvent,a as ModelStore,r as downloadVerified,e as modelsRoot,o as sha256File,t as userCacheDir,i as validateManifest};

package/lib/esm/store.js

@@ -1 +1 @@
-import"./_virtual/_rolldown/runtime.js";import{modelsRoot as e}from"./cache-dir.js";import{validateManifest as t}from"./manifest.js";import{ProgressFanout as n}from"./progress.js";import{join as r}from"node:path";import{createHash as i,randomUUID as a}from"node:crypto";import{closeSync as o,copyFileSync as s,mkdirSync as c,openSync as l,renameSync as u,statSync as d,unlinkSync as f,writeSync as p}from"node:fs";import{open as m}from"node:fs/promises";const h=/^[a-z0-9][a-z0-9-]*$/;function sizeOf(e){try{let t=d(e);return t.isFile()?t.size:null}catch{return null}}function removeQuietly(e){try{f(e)}catch{}}async function sha256File(e,t){let n=i(`sha256`),r=await m(e,`r`);try{let e=new Uint8Array(1024*1024);for(;;){t?.throwIfAborted();let{bytesRead:i}=await r.read(e,0,e.length,null);if(i===0)break;n.update(e.subarray(0,i))}}finally{await r.close()}return n.digest(`hex`)}var ModelStore=class{capability;models;defaultModel;#e;#t;#n;#r;#i=new Map;#a=new Map;constructor(e){if(!h.test(e.capability))throw Error(`ModelStore: capability '${e.capability}' is not a lowercase directory name`);if(e.models.length===0)throw Error(`ModelStore '${e.capability}': no models`);t(e.models),this.capability=e.capability,this.models=e.models,this.defaultModel=e.defaultModel??e.models[0].id,this.#e=e.root,this.#t=e.appDataDirs??[];let n=e.legacyDirs??[];this.#n=typeof n==`function`?n:()=>n,this.#r=e.fetch,this.require(this.defaultModel)}get cacheDir(){return r(this.#e??e(),this.capability)}find(e=this.defaultModel){return this.models.find(t=>t.id===e)}require(e=this.defaultModel){let t=this.find(e);if(t===void 0){let t=this.models.map(e=>e.id).join(`, `);throw RangeError(`Unknown ${this.capability} model '${e}' — known ids: ${t}`)}return t}cachePath(e=this.defaultModel){return r(this.cacheDir,this.require(e).file)}locate(e=this.defaultModel,t={}){let n=this.require(e);if(t.path!==void 0){let e=sizeOf(t.path);if(e!==n.bytes){let r=e===null?`no file`:`${e} bytes`;throw Error(`${this.capability} model '${n.id}': ${t.path} has ${r}, expected ${n.bytes}`)}return{path:t.path,source:`configured`}}for(let e of this.#t){let t=r(e,n.file);if(sizeOf(t)===n.bytes)return{path:t,source:`app-data`}}let i=r(this.cacheDir,n.file);if(sizeOf(i)===n.bytes||this.#o(n,i))return{path:i,source:`cache`}}availability(e=this.defaultModel,t={}){let n=this.require(e);return this.locate(n.id,t)===void 0?this.#i.has(n.id)?`downloading`:`downloadable`:`available`}installed(){return this.models.filter(e=>this.locate(e.id)!==void 0).map(e=>e.id)}async resolve(e=this.defaultModel,t={}){let n=this.require(e),r=this.locate(n.id,t);if(r===void 0)throw Error(`${this.capability} model '${n.id}' is not installed — await install('${n.id}') first. ${n.bytes} bytes from ${n.url}.`);return r.source!==`cache`&&await this.#s(n,r.path,t.signal),r.path}install(e=this.defaultModel,t={}){let r;try{if(r=this.require(e),!t.force&&this.locate(r.id,t)!==void 0){let e=new n(r.id);return e.add(t.monitor),this.resolve(r.id,t).then(t=>(e.report(r.bytes,r.bytes),t))}}catch(e){return Promise.reject(e)}let i=this.#i.get(r.id);if(i!==void 0)return i.progress.add(t.monitor),i.promise;let a=new n(r.id);a.add(t.monitor);let o=this.#c(r,a,t).finally(()=>this.#i.delete(r.id));return this.#i.set(r.id,{promise:o,progress:a}),o}uninstall(e=this.defaultModel){let t=this.cachePath(e);return sizeOf(t)===null?!1:(f(t),!0)}#o(e,t){for(let n of this.#n()){let i=r(n,e.file);if(sizeOf(i)===e.bytes)try{c(this.cacheDir,{recursive:!0});try{u(i,t)}catch{let e=`${t}.part-${a()}`;try{s(i,e),u(e,t)}catch(t){throw removeQuietly(e),t}removeQuietly(i)}return!0}catch{return!1}}return!1}async#s(e,t,n){let r=d(t),i=`${r.size}:${r.mtimeMs}`;if(this.#a.get(t)===i)return;let a=await sha256File(t,n);if(a!==e.sha256)throw Error(`${this.capability} model '${e.id}': ${t} has sha256 ${a}, expected ${e.sha256}`);this.#a.set(t,i)}async#c(e,t,n){let s=n.signal,d=await(n.fetch??this.#r??((e,t)=>fetch(e,t)))(e.url,{signal:s,redirect:`follow`});if(!d.ok)throw Error(`Downloading ${e.id} from ${e.url} failed: HTTP ${d.status}`);let f=d.headers.get(`content-length`);if(f!==null&&Number(f)!==e.bytes)throw Error(`Downloading ${e.id} refused: ${e.url} declares ${f} bytes, the manifest says ${e.bytes}`);let m=this.cacheDir,h=r(m,e.file);c(m,{recursive:!0});let g=`${h}.part-${a()}`,_=l(g,`w`),v=!1;try{let n=i(`sha256`),r=0,write=i=>{if(r+=i.length,r>e.bytes)throw Error(`Downloading ${e.id} failed: more than the manifest's ${e.bytes} bytes`);n.update(i),p(_,i),t.report(r,e.bytes)};t.report(0,e.bytes);let a=d.body?.getReader();if(a!==void 0)try{for(;;){s?.throwIfAborted();let{done:e,value:t}=await a.read();if(e)break;t!==void 0&&t.length>0&&write(t)}}catch(e){throw await a.cancel().catch(()=>void 0),e}else write(new Uint8Array(await d.arrayBuffer()));if(r!==e.bytes)throw Error(`Downloading ${e.id} failed: got ${r} bytes, expected ${e.bytes}`);let c=n.digest(`hex`);if(c!==e.sha256)throw Error(`Downloading ${e.id} failed: sha256 ${c}, expected ${e.sha256}`);return o(_),v=!0,u(g,h),t.report(e.bytes,e.bytes),h}catch(e){throw v||o(_),removeQuietly(g),e}}};export{ModelStore,sha256File};
+import"./_virtual/_rolldown/runtime.js";import{modelsRoot as e}from"./cache-dir.js";import{ProgressFanout as t}from"./progress.js";import{downloadVerifiedInternal as n}from"./download.js";import{validateManifest as r}from"./manifest.js";import{join as i}from"node:path";import{createHash as a,randomUUID as o}from"node:crypto";import{copyFileSync as s,mkdirSync as c,renameSync as l,statSync as u,unlinkSync as d}from"node:fs";import{open as f}from"node:fs/promises";const p=/^[a-z0-9][a-z0-9-]*$/;function sizeOf(e){try{let t=u(e);return t.isFile()?t.size:null}catch{return null}}function removeQuietly(e){try{d(e)}catch{}}async function sha256File(e,t){let n=a(`sha256`),r=await f(e,`r`);try{let e=new Uint8Array(1024*1024);for(;;){t?.throwIfAborted();let{bytesRead:i}=await r.read(e,0,e.length,null);if(i===0)break;n.update(e.subarray(0,i))}}finally{await r.close()}return n.digest(`hex`)}var ModelStore=class{capability;models;defaultModel;#e;#t;#n;#r;#i=new Map;#a=new Map;constructor(e){if(!p.test(e.capability))throw Error(`ModelStore: capability '${e.capability}' is not a lowercase directory name`);if(e.models.length===0)throw Error(`ModelStore '${e.capability}': no models`);r(e.models),this.capability=e.capability,this.models=e.models,this.defaultModel=e.defaultModel??e.models[0].id,this.#e=e.root,this.#t=e.appDataDirs??[];let t=e.legacyDirs??[];this.#n=typeof t==`function`?t:()=>t,this.#r=e.fetch,this.require(this.defaultModel)}get cacheDir(){return i(this.#e??e(),this.capability)}find(e=this.defaultModel){return this.models.find(t=>t.id===e)}require(e=this.defaultModel){let t=this.find(e);if(t===void 0){let t=this.models.map(e=>e.id).join(`, `);throw RangeError(`Unknown ${this.capability} model '${e}' — known ids: ${t}`)}return t}cachePath(e=this.defaultModel){return i(this.cacheDir,this.require(e).file)}locate(e=this.defaultModel,t={}){let n=this.require(e);if(t.path!==void 0){let e=sizeOf(t.path);if(e!==n.bytes){let r=e===null?`no file`:`${e} bytes`;throw Error(`${this.capability} model '${n.id}': ${t.path} has ${r}, expected ${n.bytes}`)}return{path:t.path,source:`configured`}}for(let e of this.#t){let t=i(e,n.file);if(sizeOf(t)===n.bytes)return{path:t,source:`app-data`}}let r=i(this.cacheDir,n.file);if(sizeOf(r)===n.bytes||this.#o(n,r))return{path:r,source:`cache`}}availability(e=this.defaultModel,t={}){let n=this.require(e);return this.locate(n.id,t)===void 0?this.#i.has(n.id)?`downloading`:`downloadable`:`available`}installed(){return this.models.filter(e=>this.locate(e.id)!==void 0).map(e=>e.id)}async resolve(e=this.defaultModel,t={}){let n=this.require(e),r=this.locate(n.id,t);if(r===void 0)throw Error(`${this.capability} model '${n.id}' is not installed — await install('${n.id}') first. ${n.bytes} bytes from ${n.url}.`);return r.source!==`cache`&&await this.#s(n,r.path,t.signal),r.path}install(e=this.defaultModel,n={}){let r;try{if(r=this.require(e),!n.force&&this.locate(r.id,n)!==void 0){let e=new t(r.id);return e.add(n.monitor),this.resolve(r.id,n).then(t=>(e.report(r.bytes,r.bytes),t))}}catch(e){return Promise.reject(e)}let i=this.#i.get(r.id);if(i!==void 0)return i.progress.add(n.monitor),i.promise;let a=new t(r.id);a.add(n.monitor);let o=this.#c(r,a,n).finally(()=>this.#i.delete(r.id));return this.#i.set(r.id,{promise:o,progress:a}),o}uninstall(e=this.defaultModel){let t=this.cachePath(e);return sizeOf(t)===null?!1:(d(t),!0)}#o(e,t){for(let n of this.#n()){let r=i(n,e.file);if(sizeOf(r)===e.bytes)try{c(this.cacheDir,{recursive:!0});try{l(r,t)}catch{let e=`${t}.part-${o()}`;try{s(r,e),l(e,t)}catch(t){throw removeQuietly(e),t}removeQuietly(r)}return!0}catch{return!1}}return!1}async#s(e,t,n){let r=u(t),i=`${r.size}:${r.mtimeMs}`;if(this.#a.get(t)===i)return;let a=await sha256File(t,n);if(a!==e.sha256)throw Error(`${this.capability} model '${e.id}': ${t} has sha256 ${a}, expected ${e.sha256}`);this.#a.set(t,i)}async#c(e,t,r){let a=i(this.cacheDir,e.file);return await n({url:e.url,dest:a,bytes:e.bytes,sha256:e.sha256,signal:r.signal,fetch:r.fetch??this.#r,progress:t,id:e.id}),a}};export{ModelStore,sha256File};

package/lib/types/download.d.ts

@@ -0,0 +1,63 @@
+import { CreateMonitorCallback, ProgressFanout } from './progress.js';
+/** The subset of `fetch` the downloader uses, so a test can inject one. */
+export type FetchLike = (url: string, init: {
+    signal?: AbortSignal;
+    redirect?: 'follow';
+    headers?: Record<string, string>;
+}) => Promise<{
+    ok: boolean;
+    status: number;
+    headers: {
+        get(name: string): string | null;
+    };
+    body?: {
+        getReader(): {
+            read(): Promise<{
+                done: boolean;
+                value?: Uint8Array;
+            }>;
+            cancel(): Promise<void>;
+        };
+    } | null;
+    arrayBuffer(): Promise<ArrayBuffer>;
+}>;
+export interface DownloadVerifiedOptions {
+    /** URL to fetch. */
+    url: string;
+    /** Final path; parent directories are created. */
+    dest: string;
+    /** Expected size in bytes. Checked while streaming and after. */
+    bytes?: number;
+    /** Expected SHA-256, lowercase hex. Checked after the stream finishes. */
+    sha256?: string;
+    /** Extra request headers. */
+    headers?: Record<string, string>;
+    /** Receives an EventTarget firing `downloadprogress`, at least at 0 and 1. */
+    monitor?: CreateMonitorCallback;
+    /** Abort the request and close the partial file. */
+    signal?: AbortSignal;
+    /** Override the global `fetch`. */
+    fetch?: FetchLike;
+}
+interface DownloadVerifiedInternalOptions extends DownloadVerifiedOptions {
+    /** Identifier used in error messages. Omit for the standalone public function. */
+    id?: string;
+    /** Existing progress fanout (e.g. ModelStore's shared one). */
+    progress?: ProgressFanout;
+}
+/** @internal Variant that lets ModelStore keep its existing error wording. */
+export declare function downloadVerifiedInternal(options: DownloadVerifiedInternalOptions): Promise<{
+    bytes: number;
+    sha256: string;
+}>;
+/**
+ * Stream `url` to `dest`, verifying size and/or SHA-256 while downloading.
+ * The file is written under a temporary name and renamed into place only after
+ * verification. Returns the measured size and digest. Removes the partial file
+ * on any failure or abort.
+ */
+export declare function downloadVerified(options: DownloadVerifiedOptions): Promise<{
+    bytes: number;
+    sha256: string;
+}>;
+export {};

package/lib/types/download.spec.d.ts

@@ -0,0 +1,2 @@
+declare const _default: () => Promise<void>;
+export default _default;

package/lib/types/index.d.ts

@@ -1,4 +1,5 @@
 export { modelsRoot, userCacheDir } from './cache-dir.js';
+export { downloadVerified, type DownloadVerifiedOptions, type FetchLike } from './download.js';
 export { validateManifest, type ModelEntry } from './manifest.js';
 export { DownloadProgressEvent, type CreateMonitorCallback } from './progress.js';
-export { ModelStore, sha256File, type FetchLike, type InstallOptions, type LocatedModel, type LocateOptions, type ModelAvailability, type ModelSource, type ModelStoreOptions, type ResolveOptions, } from './store.js';
+export { ModelStore, sha256File, type InstallOptions, type LocatedModel, type LocateOptions, type ModelAvailability, type ModelSource, type ModelStoreOptions, type ResolveOptions, } from './store.js';

package/lib/types/store.d.ts

@@ -1,5 +1,7 @@
+import { type FetchLike } from './download.js';
 import { type ModelEntry } from './manifest.js';
 import { type CreateMonitorCallback } from './progress.js';
+export { type FetchLike };
 /**
  * The WebML CG availability states. The store itself never answers
  * `unavailable` — that is the capability's verdict (no engine, or `checkFit`
@@ -12,27 +14,6 @@ export interface LocatedModel {
     readonly path: string;
     readonly source: ModelSource;
 }
-/** The subset of `fetch` the store uses, so a test can inject one. */
-export type FetchLike = (url: string, init: {
-    signal?: AbortSignal;
-    redirect?: 'follow';
-}) => Promise<{
-    ok: boolean;
-    status: number;
-    headers: {
-        get(name: string): string | null;
-    };
-    body?: {
-        getReader(): {
-            read(): Promise<{
-                done: boolean;
-                value?: Uint8Array;
-            }>;
-            cancel(): Promise<void>;
-        };
-    } | null;
-    arrayBuffer(): Promise<ArrayBuffer>;
-}>;
 export interface ModelStoreOptions<E extends ModelEntry> {
     /** Directory name under the models root: `speech`, `embeddings`, `image`. Lowercase, digits, dashes. */
     readonly capability: string;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@gjsify/model-store",
-    "version": "0.64.25",
+    "version": "0.64.26-next.312",
     "description": "Where an AI model lives and how it gets there: one store per capability with streaming download, SHA-256 verification, WebML-shaped download progress, and three sources in order (a configured path, an app's read-only data dir, the per-user cache). Pure TypeScript on node:* APIs.",
     "module": "lib/esm/index.js",
     "types": "lib/types/index.d.ts",
@@ -39,15 +39,15 @@
         "gguf"
     ],
     "dependencies": {
-        "@gjsify/crypto": "^0.64.25",
-        "@gjsify/fs": "^0.64.25",
-        "@gjsify/os": "^0.64.25",
-        "@gjsify/path": "^0.64.25",
-        "@gjsify/process": "^0.64.25"
+        "@gjsify/crypto": "0.64.26-next.312",
+        "@gjsify/fs": "0.64.26-next.312",
+        "@gjsify/os": "0.64.26-next.312",
+        "@gjsify/path": "0.64.26-next.312",
+        "@gjsify/process": "0.64.26-next.312"
     },
     "devDependencies": {
-        "@gjsify/cli": "^0.64.25",
-        "@gjsify/unit": "^0.64.25",
+        "@gjsify/cli": "0.64.26-next.312",
+        "@gjsify/unit": "0.64.26-next.312",
         "@types/node": "^25.9.2",
         "typescript": "^6.0.3"
     },
@@ -78,5 +78,6 @@
             "win32": "The cache root is %LOCALAPPDATA%\\gjsify\\models, where GLib's get_user_cache_dir() would answer the Internet cache folder."
         },
         "headless": true
-    }
+    },
+    "gitHead": "75fa6155a8f1fa0f7e5d70c43418ae41ecac8c0e"
 }