@hakubase/cli 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -58,7 +58,7 @@ npx hakubase deploy dist --publish --message "New home"
58
58
  npx hakubase publish 12 # put an uploaded version online
59
59
  ```
60
60
 
61
- `deploy` zips the folder (up to 32 MB), adds `hakubase.json` from the project root if the folder does not have one, and uploads it as a new version. Without `--publish` the version stays in preview and the command prints its preview link; with it, the version goes online.
61
+ `deploy` sends the list of the folder's files with their fingerprints, uploads only the ones Hakubase does not have yet (in groups, up to 500 MB per site and 10 MB per file, videos included), reads `hakubase.json` from the folder or the project root, and closes a new version. Without `--publish` the version stays in preview and the command prints its preview link; with it, the version goes online.
62
62
 
63
63
  Deploying needs a **publishing key**.
64
64
 
package/bin/hakubase.js CHANGED
@@ -1,5 +1,8 @@
1
1
  #!/usr/bin/env node
2
2
  import { main } from '../lib/commands.js'
3
+ import { installLocalhostResolver } from '../lib/localhost.js'
4
+
5
+ installLocalhostResolver()
3
6
 
4
7
  main(process.argv.slice(2)).then(
5
8
  (code) => { process.exitCode = code },
package/lib/commands.js CHANGED
@@ -2,6 +2,7 @@
2
2
  // goes through `io` so the tests can read it.
3
3
 
4
4
  import { execFileSync } from 'node:child_process'
5
+ import crypto from 'node:crypto'
5
6
  import fs from 'node:fs'
6
7
  import path from 'node:path'
7
8
 
@@ -9,9 +10,10 @@ import {
9
10
  ensureGitignored, findRoot, maskKey, readProject, resolveKey, writeEnvKey, writeProject,
10
11
  } from './project.js'
11
12
  import { generateTypes } from './types.js'
12
- import { createZip } from './zip.js'
13
13
 
14
- export const MAX_ZIP = 32 * 1024 * 1024
14
+ // One request carries at most this much: the server takes 32 MB per upload.
15
+ export const BATCH_BYTES = 24 * 1024 * 1024
16
+ export const BATCH_FILES = 200
15
17
  export const BUILD_DIRS = ['dist', 'build', 'out']
16
18
  // What an operating system leaves in a folder, never part of a site.
17
19
  const JUNK = new Set(['.DS_Store', 'Thumbs.db', 'desktop.ini'])
@@ -87,9 +89,13 @@ function detailOf(body, status) {
87
89
  return `HTTP ${status}`
88
90
  }
89
91
 
90
- export async function request(ctx, method, url, { key, body } = {}) {
92
+ export async function request(ctx, method, url, { key, body, json } = {}) {
91
93
  const headers = { accept: 'application/json', 'user-agent': 'hakubase-cli' }
92
94
  if (key) headers.authorization = `Bearer ${key}`
95
+ if (json !== undefined) {
96
+ headers['content-type'] = 'application/json'
97
+ body = JSON.stringify(json)
98
+ }
93
99
  let response
94
100
  try {
95
101
  response = await ctx.fetch(url, { method, headers, body })
@@ -162,7 +168,7 @@ export function buildDir(cwd, given) {
162
168
  throw new CliError('No built folder: build the site first, or say which folder to deploy (dist, build, out).')
163
169
  }
164
170
 
165
- /** Every file of the folder, as zip entries with forward-slash names. */
171
+ /** Every file of the folder, with forward-slash names. */
166
172
  export function collectFiles(dir) {
167
173
  const entries = []
168
174
  const walk = (current, prefix) => {
@@ -192,31 +198,79 @@ function gitSubject(cwd) {
192
198
  }
193
199
  }
194
200
 
201
+ /** The list Hakubase reads instead of a zip: path, fingerprint and size of every file. */
202
+ export function listOf(entries) {
203
+ return entries.map((e) => ({
204
+ path: '/' + e.name,
205
+ hash: crypto.createHash('sha256').update(e.data).digest('hex'),
206
+ size: e.data.length,
207
+ data: e.data,
208
+ }))
209
+ }
210
+
211
+ /** The missing files in groups that fit one request each. */
212
+ export function batches(files) {
213
+ const out = []
214
+ let current = []
215
+ let bytes = 0
216
+ for (const file of files) {
217
+ if (current.length && (bytes + file.size > BATCH_BYTES || current.length >= BATCH_FILES)) {
218
+ out.push(current)
219
+ current = []
220
+ bytes = 0
221
+ }
222
+ current.push(file)
223
+ bytes += file.size
224
+ }
225
+ if (current.length) out.push(current)
226
+ return out
227
+ }
228
+
229
+ function readConfig(root, entries) {
230
+ const inside = entries.find((e) => e.name === 'hakubase.json')
231
+ const file = path.join(root, 'hakubase.json')
232
+ const raw = inside ? inside.data : (fs.existsSync(file) ? fs.readFileSync(file) : null)
233
+ if (!raw) return null
234
+ try {
235
+ return JSON.parse(raw.toString('utf8'))
236
+ } catch {
237
+ throw new CliError('hakubase.json is not JSON.')
238
+ }
239
+ }
240
+
195
241
  export async function deploy(ctx, positional, flags) {
196
242
  const t = target(ctx, flags)
197
243
  const dir = buildDir(ctx.cwd, positional[0])
198
244
  const entries = collectFiles(dir)
199
245
  if (!entries.length) throw new CliError(`${path.relative(ctx.cwd, dir) || dir} is empty.`)
200
- const manifest = path.join(t.root, 'hakubase.json')
201
- if (!entries.some((e) => e.name === 'hakubase.json') && fs.existsSync(manifest)) {
202
- entries.push({ name: 'hakubase.json', data: fs.readFileSync(manifest), mtime: fs.statSync(manifest).mtime })
203
- }
204
- const zip = createZip(entries)
205
- if (zip.length > MAX_ZIP) {
206
- throw new CliError(`The zip is ${megabytes(zip.length)}: Hakubase takes up to 32 MB. Put the videos in the File library.`)
246
+ const config = readConfig(t.root, entries)
247
+ const listed = listOf(entries.filter((e) => e.name !== 'hakubase.json'))
248
+ const files = listed.map(({ path: p, hash, size }) => ({ path: p, hash, size }))
249
+ const base = `${t.api}/api/sites/${encodeURIComponent(t.site)}/code-site`
250
+ // 1. What Hakubase lacks: a deploy that changed one page sends one page.
251
+ const plan = await request(ctx, 'POST', `${base}/uploads`, { key: t.key, json: { files, config } })
252
+ const missing = new Set(plan.missing || [])
253
+ const unique = new Map()
254
+ for (const file of listed) if (missing.has(file.hash) && !unique.has(file.hash)) unique.set(file.hash, file)
255
+ const groups = batches([...unique.values()])
256
+ // 2. The missing files, a group at a time, each named by its fingerprint.
257
+ let sent = 0
258
+ for (const group of groups) {
259
+ const form = new FormData()
260
+ for (const file of group) form.append('files', new Blob([file.data], { type: 'application/octet-stream' }), file.hash)
261
+ await request(ctx, 'POST', `${base}/files`, { key: t.key, body: form })
262
+ sent += group.reduce((n, f) => n + f.size, 0)
263
+ if (groups.length > 1) ctx.out(` ${megabytes(sent)} of ${megabytes(plan.missing_bytes || sent)}`)
207
264
  }
265
+ // 3. The version, once every file is there.
208
266
  const github = Boolean(ctx.env.GITHUB_SHA)
209
267
  const commit = typeof flags.commit === 'string' ? flags.commit : (ctx.env.GITHUB_SHA || '')
210
268
  const via = typeof flags.via === 'string' ? flags.via : (github ? 'github' : 'cli')
211
269
  const message = typeof flags.message === 'string' ? flags.message : (github ? gitSubject(ctx.cwd) : '')
212
- const form = new FormData()
213
- form.append('file', new Blob([zip], { type: 'application/zip' }), 'site.zip')
214
- form.append('publish', flags.publish ? 'true' : 'false')
215
- form.append('message', message)
216
- form.append('commit', commit)
217
- form.append('via', via)
218
- const version = await request(ctx, 'POST', `${t.api}/api/sites/${encodeURIComponent(t.site)}/code-site/versions`, { key: t.key, body: form })
219
- ctx.out(`Version ${version.number} uploaded (${version.files} files, ${megabytes(version.size || 0)}, ${megabytes(version.uploaded || 0)} new).`)
270
+ const version = await request(ctx, 'POST', `${base}/versions/manifest`, {
271
+ key: t.key, json: { files, config, publish: Boolean(flags.publish), message, commit, via },
272
+ })
273
+ ctx.out(`Version ${version.number} uploaded (${version.files} files, ${megabytes(version.size || 0)}, ${megabytes(sent)} new).`)
220
274
  if (version.online) {
221
275
  ctx.out(`Online: ${t.siteUrl || await siteUrl(ctx, t)}`)
222
276
  } else {
@@ -0,0 +1,30 @@
1
+ // `*.localhost` is the loopback by definition (RFC 6761), and browsers
2
+ // treat it so; Node asks the system resolver, which on macOS does not.
3
+ // A Hakubase in development lives on `*.hakubase.localhost`: without
4
+ // this, `hakubase link` and `deploy` against it fail with ENOTFOUND.
5
+ import dns from 'node:dns'
6
+
7
+ const original = dns.lookup
8
+
9
+ export function isLoopbackName(host) {
10
+ return typeof host === 'string' && /(^|\.)localhost\.?$/i.test(host)
11
+ }
12
+
13
+ export function installLocalhostResolver() {
14
+ if (dns.lookup.__hakubase) return
15
+ const lookup = function (host, options, callback) {
16
+ if (typeof options === 'function') {
17
+ callback = options
18
+ options = {}
19
+ }
20
+ if (isLoopbackName(host)) {
21
+ const all = typeof options === 'object' && options && options.all
22
+ return all
23
+ ? process.nextTick(callback, null, [{ address: '127.0.0.1', family: 4 }])
24
+ : process.nextTick(callback, null, '127.0.0.1', 4)
25
+ }
26
+ return original.call(dns, host, options, callback)
27
+ }
28
+ lookup.__hakubase = true
29
+ dns.lookup = lookup
30
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hakubase/cli",
3
- "version": "0.1.0",
3
+ "version": "0.2.0",
4
4
  "description": "Link, type, develop and deploy Hakubase code sites; Vite and Astro plugins for development with real data.",
5
5
  "type": "module",
6
6
  "bin": {
package/lib/zip.js DELETED
@@ -1,105 +0,0 @@
1
- // A minimal zip writer: deflate or store, UTF-8 names, no zip64.
2
- // Enough for a built site, which Hakubase caps well below the limits
3
- // of the classic format (2,000 files, 100 MB uncompressed).
4
-
5
- import zlib from 'node:zlib'
6
-
7
- let table = null
8
-
9
- function crcTable() {
10
- if (table) return table
11
- table = new Uint32Array(256)
12
- for (let n = 0; n < 256; n++) {
13
- let c = n
14
- for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1
15
- table[n] = c >>> 0
16
- }
17
- return table
18
- }
19
-
20
- export function crc32(data) {
21
- if (typeof zlib.crc32 === 'function') return zlib.crc32(data) >>> 0
22
- const t = crcTable()
23
- let c = 0xffffffff
24
- for (let i = 0; i < data.length; i++) c = t[(c ^ data[i]) & 0xff] ^ (c >>> 8)
25
- return (c ^ 0xffffffff) >>> 0
26
- }
27
-
28
- function dosDateTime(date) {
29
- const d = date instanceof Date && !Number.isNaN(date.getTime()) ? date : new Date()
30
- const year = Math.max(1980, d.getFullYear())
31
- const time = (d.getHours() << 11) | (d.getMinutes() << 5) | Math.floor(d.getSeconds() / 2)
32
- const day = ((year - 1980) << 9) | ((d.getMonth() + 1) << 5) | d.getDate()
33
- return { time, day }
34
- }
35
-
36
- const UTF8 = 0x0800
37
-
38
- /**
39
- * entries: [{ name: 'assets/app.js', data: Buffer, mtime?: Date }]
40
- * Returns the zip as one Buffer.
41
- */
42
- export function createZip(entries) {
43
- if (entries.length > 0xffff) throw new Error('Too many files for a zip without zip64.')
44
- const locals = []
45
- const centrals = []
46
- let offset = 0
47
- for (const entry of entries) {
48
- const name = Buffer.from(String(entry.name).replace(/\\/g, '/').replace(/^\/+/, ''), 'utf8')
49
- const data = Buffer.isBuffer(entry.data) ? entry.data : Buffer.from(entry.data)
50
- const crc = crc32(data)
51
- const deflated = zlib.deflateRawSync(data)
52
- const stored = deflated.length >= data.length
53
- const body = stored ? data : deflated
54
- const method = stored ? 0 : 8
55
- const { time, day } = dosDateTime(entry.mtime)
56
-
57
- const local = Buffer.alloc(30)
58
- local.writeUInt32LE(0x04034b50, 0)
59
- local.writeUInt16LE(20, 4)
60
- local.writeUInt16LE(UTF8, 6)
61
- local.writeUInt16LE(method, 8)
62
- local.writeUInt16LE(time, 10)
63
- local.writeUInt16LE(day, 12)
64
- local.writeUInt32LE(crc, 14)
65
- local.writeUInt32LE(body.length, 18)
66
- local.writeUInt32LE(data.length, 22)
67
- local.writeUInt16LE(name.length, 26)
68
- local.writeUInt16LE(0, 28)
69
-
70
- const central = Buffer.alloc(46)
71
- central.writeUInt32LE(0x02014b50, 0)
72
- central.writeUInt16LE(20, 4)
73
- central.writeUInt16LE(20, 6)
74
- central.writeUInt16LE(UTF8, 8)
75
- central.writeUInt16LE(method, 10)
76
- central.writeUInt16LE(time, 12)
77
- central.writeUInt16LE(day, 14)
78
- central.writeUInt32LE(crc, 16)
79
- central.writeUInt32LE(body.length, 20)
80
- central.writeUInt32LE(data.length, 24)
81
- central.writeUInt16LE(name.length, 28)
82
- central.writeUInt16LE(0, 30)
83
- central.writeUInt16LE(0, 32)
84
- central.writeUInt16LE(0, 34)
85
- central.writeUInt16LE(0, 36)
86
- central.writeUInt32LE(0, 38)
87
- central.writeUInt32LE(offset, 42)
88
-
89
- locals.push(local, name, body)
90
- centrals.push(central, name)
91
- offset += local.length + name.length + body.length
92
- if (offset > 0xffffffff) throw new Error('The zip is too big for a zip without zip64.')
93
- }
94
- const directory = Buffer.concat(centrals)
95
- const end = Buffer.alloc(22)
96
- end.writeUInt32LE(0x06054b50, 0)
97
- end.writeUInt16LE(0, 4)
98
- end.writeUInt16LE(0, 6)
99
- end.writeUInt16LE(entries.length, 8)
100
- end.writeUInt16LE(entries.length, 10)
101
- end.writeUInt32LE(directory.length, 12)
102
- end.writeUInt32LE(offset, 16)
103
- end.writeUInt16LE(0, 20)
104
- return Buffer.concat([...locals, directory, end])
105
- }