@qvac/registry-client 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +4 -0
- package/README.md +13 -1
- package/lib/client.js +85 -10
- package/lib/config.js +5 -2
- package/package.json +3 -2
package/NOTICE
ADDED
package/README.md
CHANGED
|
@@ -110,8 +110,20 @@ result.artifact.stream.pipe(fs.createWriteStream('./model.ggml'))
|
|
|
110
110
|
|
|
111
111
|
The package includes a CLI for querying and downloading models from the registry.
|
|
112
112
|
|
|
113
|
+
### Install
|
|
114
|
+
|
|
115
|
+
The package is hosted on GitHub Packages. Configure npm to use the GitHub registry for the `@tetherto` scope, then install globally:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
echo "@tetherto:registry=https://npm.pkg.github.com" >> ~/.npmrc
|
|
119
|
+
echo "//npm.pkg.github.com/:_authToken=YOUR_GITHUB_PAT" >> ~/.npmrc
|
|
120
|
+
npm install -g @tetherto/registry-client-mono
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Verify installation:
|
|
124
|
+
|
|
113
125
|
```bash
|
|
114
|
-
|
|
126
|
+
qvac-registry --help
|
|
115
127
|
```
|
|
116
128
|
|
|
117
129
|
### List models
|
package/lib/client.js
CHANGED
|
@@ -151,6 +151,23 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
151
151
|
}
|
|
152
152
|
}
|
|
153
153
|
|
|
154
|
+
async _checkBlobProgress (core, blobPointer) {
|
|
155
|
+
const totalBlocks = blobPointer.blockLength
|
|
156
|
+
const totalBytes = blobPointer.byteLength
|
|
157
|
+
|
|
158
|
+
if (totalBlocks === 0) {
|
|
159
|
+
return { cachedBlocks: 0, totalBlocks: 0, totalBytes: 0 }
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
let cachedBlocks = 0
|
|
163
|
+
for (let i = blobPointer.blockOffset; i < blobPointer.blockOffset + totalBlocks; i++) {
|
|
164
|
+
const hasBlock = await core.has(i)
|
|
165
|
+
if (hasBlock) cachedBlocks++
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
return { cachedBlocks, totalBlocks, totalBytes }
|
|
169
|
+
}
|
|
170
|
+
|
|
154
171
|
async _getBlobsCore (blobsCoreKey) {
|
|
155
172
|
const keyBuffer = Buffer.isBuffer(blobsCoreKey)
|
|
156
173
|
? blobsCoreKey
|
|
@@ -208,10 +225,12 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
208
225
|
await core.update({ wait: true })
|
|
209
226
|
this.logger.debug('Blobs core updated')
|
|
210
227
|
|
|
228
|
+
const totalSize = model.blobBinding.byteLength
|
|
229
|
+
|
|
211
230
|
let artifact
|
|
212
231
|
if (options.outputFile) {
|
|
213
|
-
await this._streamBlobToFile(blobs, model.blobBinding, options.outputFile, options)
|
|
214
|
-
artifact = { path: options.outputFile }
|
|
232
|
+
await this._streamBlobToFile(blobs, core, model.blobBinding, options.outputFile, options)
|
|
233
|
+
artifact = { path: options.outputFile, totalSize }
|
|
215
234
|
|
|
216
235
|
if (blobs) await blobs.close()
|
|
217
236
|
if (core) await core.close()
|
|
@@ -220,7 +239,7 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
220
239
|
wait: true,
|
|
221
240
|
timeout: options.timeout || 30000
|
|
222
241
|
})
|
|
223
|
-
artifact = { stream }
|
|
242
|
+
artifact = { stream, totalSize }
|
|
224
243
|
|
|
225
244
|
const cleanup = async () => {
|
|
226
245
|
if (blobs) {
|
|
@@ -271,7 +290,45 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
271
290
|
}
|
|
272
291
|
}
|
|
273
292
|
|
|
274
|
-
async _streamBlobToFile (blobs, blobPointer, filePath, options) {
|
|
293
|
+
async _streamBlobToFile (blobs, core, blobPointer, filePath, options) {
|
|
294
|
+
const { cachedBlocks, totalBlocks, totalBytes } = await this._checkBlobProgress(core, blobPointer)
|
|
295
|
+
|
|
296
|
+
this.logger.debug('Blob progress before download', {
|
|
297
|
+
cachedBlocks,
|
|
298
|
+
totalBlocks,
|
|
299
|
+
totalBytes,
|
|
300
|
+
resuming: cachedBlocks > 0 && cachedBlocks < totalBlocks
|
|
301
|
+
})
|
|
302
|
+
|
|
303
|
+
const bytesPerBlock = totalBlocks > 0 ? totalBytes / totalBlocks : 0
|
|
304
|
+
let downloadedBytes = Math.floor(cachedBlocks * bytesPerBlock)
|
|
305
|
+
|
|
306
|
+
if (options.onProgress && downloadedBytes > 0) {
|
|
307
|
+
options.onProgress({
|
|
308
|
+
downloaded: downloadedBytes,
|
|
309
|
+
total: totalBytes,
|
|
310
|
+
cachedBlocks,
|
|
311
|
+
totalBlocks
|
|
312
|
+
})
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
const progressHandler = (index, bytes) => {
|
|
316
|
+
if (index >= blobPointer.blockOffset && index < blobPointer.blockOffset + blobPointer.blockLength) {
|
|
317
|
+
downloadedBytes += bytes
|
|
318
|
+
const capped = Math.min(downloadedBytes, totalBytes)
|
|
319
|
+
if (options.onProgress) {
|
|
320
|
+
options.onProgress({
|
|
321
|
+
downloaded: capped,
|
|
322
|
+
total: totalBytes,
|
|
323
|
+
cachedBlocks,
|
|
324
|
+
totalBlocks
|
|
325
|
+
})
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
core.on('download', progressHandler)
|
|
331
|
+
|
|
275
332
|
const stream = blobs.createReadStream(blobPointer, {
|
|
276
333
|
wait: true,
|
|
277
334
|
timeout: options.timeout || 30000
|
|
@@ -282,12 +339,30 @@ class QVACRegistryClient extends ReadyResource {
|
|
|
282
339
|
|
|
283
340
|
const writeStream = fs.createWriteStream(filePath)
|
|
284
341
|
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
342
|
+
try {
|
|
343
|
+
await new Promise((resolve, reject) => {
|
|
344
|
+
stream.pipe(writeStream)
|
|
345
|
+
writeStream.on('finish', resolve)
|
|
346
|
+
writeStream.on('error', reject)
|
|
347
|
+
stream.on('error', reject)
|
|
348
|
+
|
|
349
|
+
if (options.signal) {
|
|
350
|
+
if (options.signal.aborted) {
|
|
351
|
+
stream.destroy()
|
|
352
|
+
writeStream.destroy()
|
|
353
|
+
reject(new Error('Download cancelled'))
|
|
354
|
+
return
|
|
355
|
+
}
|
|
356
|
+
options.signal.addEventListener('abort', () => {
|
|
357
|
+
stream.destroy()
|
|
358
|
+
writeStream.destroy()
|
|
359
|
+
reject(new Error('Download cancelled'))
|
|
360
|
+
}, { once: true })
|
|
361
|
+
}
|
|
362
|
+
})
|
|
363
|
+
} finally {
|
|
364
|
+
core.off('download', progressHandler)
|
|
365
|
+
}
|
|
291
366
|
}
|
|
292
367
|
|
|
293
368
|
async _close () {
|
package/lib/config.js
CHANGED
|
@@ -6,6 +6,8 @@ const Logger = require('./logger')
|
|
|
6
6
|
const os = require('os')
|
|
7
7
|
const path = require('path')
|
|
8
8
|
|
|
9
|
+
const DEFAULT_REGISTRY_CORE_KEY = 'u6pq8h3kof7ck9g6kjusykfxaxqaqtnoydq15hhyuzrf55nt384y'
|
|
10
|
+
|
|
9
11
|
class RegistryConfig {
|
|
10
12
|
constructor (opts = {}) {
|
|
11
13
|
this.logger = new Logger(opts.logger)
|
|
@@ -38,8 +40,9 @@ class RegistryConfig {
|
|
|
38
40
|
}
|
|
39
41
|
|
|
40
42
|
const envKey = getEnv(ENV_KEYS.QVAC_REGISTRY_CORE_KEY)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
+
const result = envKey || DEFAULT_REGISTRY_CORE_KEY
|
|
44
|
+
this.logger.debug('getRegistryCoreKey resolved', { fromEnv: !!envKey, usingDefault: !envKey })
|
|
45
|
+
return result
|
|
43
46
|
}
|
|
44
47
|
}
|
|
45
48
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/registry-client",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.6",
|
|
4
4
|
"description": "QVAC Registry client library for read-only queries via Hyperswarm",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "commonjs",
|
|
@@ -19,7 +19,8 @@
|
|
|
19
19
|
"lib/",
|
|
20
20
|
"utils/",
|
|
21
21
|
"README.md",
|
|
22
|
-
"LICENSE"
|
|
22
|
+
"LICENSE",
|
|
23
|
+
"NOTICE"
|
|
23
24
|
],
|
|
24
25
|
"scripts": {
|
|
25
26
|
"test:unit": "brittle tests/unit/*.test.js",
|