dsh-usage-badge 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -3
- package/docs/design-notes.md +389 -1
- package/lib/buckets.js +237 -5
- package/lib/cache-store.js +574 -129
- package/lib/client.js +348 -3
- package/lib/fold.js +20 -10
- package/lib/index.js +469 -5
- package/lib/sync.js +1179 -0
- package/lib/webdav.js +532 -0
- package/package.json +3 -1
package/lib/sync.js
ADDED
|
@@ -0,0 +1,1179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-usage-badge — the shard format and the two directions of a sync.
|
|
3
|
+
*
|
|
4
|
+
* The cache's own store stays local; this module is the **export/import boundary**. It lives apart
|
|
5
|
+
* from the store because it needs two things the store must not know about:
|
|
6
|
+
*
|
|
7
|
+
* - **a wire format.** One shard is one calendar month of one device's rows, as NDJSON compressed
|
|
8
|
+
* with `zstd` (`node:zlib`, already used to read `.jsonl.zstd` logs). A month is the granularity
|
|
9
|
+
* because a day would mean hundreds of files and a request each, while a year would mean
|
|
10
|
+
* re-uploading ~27 MB whenever any single day inside it changed.
|
|
11
|
+
* - **a manifest.** Each device publishes **its own** `index.json`, listing the shards it wrote
|
|
12
|
+
* with a sha256 each. There is deliberately no shared manifest: a file two devices both wrote
|
|
13
|
+
* would need locking, and this design has no locking.
|
|
14
|
+
*
|
|
15
|
+
* ## What a device must never publish
|
|
16
|
+
*
|
|
17
|
+
* Only rows with `source = local` are exported. A row that *arrived* from another device is stored so
|
|
18
|
+
* the panel can count it, but exporting it back would make this device claim another's history — and
|
|
19
|
+
* if that other device then deleted it, the copy here would resurrect it forever.
|
|
20
|
+
*
|
|
21
|
+
* `fallback` rows are not exported either: they are copies of a log version this device has already
|
|
22
|
+
* moved past, kept as local insurance rather than as history to publish.
|
|
23
|
+
*
|
|
24
|
+
* ## The import side
|
|
25
|
+
*
|
|
26
|
+
* A shard's rows carry their own `source` (the id of the device that wrote them) and their own
|
|
27
|
+
* stamp, and are handed to the store as they are. The store then applies the rule it applies to any
|
|
28
|
+
* weak copy: a row that is not strictly fuller than the one already held for that `(session, date,
|
|
29
|
+
* source, slot)` is dropped. That is what stops a machine whose logs were pruned from overwriting the
|
|
30
|
+
* fuller history another machine still has.
|
|
31
|
+
*
|
|
32
|
+
* @module dsh-usage-badge/sync
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { createHash } from 'node:crypto'
|
|
36
|
+
import { mkdirSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs'
|
|
37
|
+
import { hostname } from 'node:os'
|
|
38
|
+
import { join } from 'node:path'
|
|
39
|
+
import { zstdCompressSync, zstdDecompressSync } from 'node:zlib'
|
|
40
|
+
|
|
41
|
+
import { CURRENT, FALLBACK, LOCAL_SOURCE, rowKind, rowSource } from './buckets.js'
|
|
42
|
+
import { WebDavClient } from './webdav.js'
|
|
43
|
+
|
|
44
|
+
/** The shard format this build writes. A shard of another version is refused, not guessed at. */
|
|
45
|
+
export const SHARD_VERSION = 1
|
|
46
|
+
|
|
47
|
+
/** The manifest format this build writes. */
|
|
48
|
+
export const MANIFEST_VERSION = 1
|
|
49
|
+
|
|
50
|
+
/** The file extension a shard carries; the file name is its month. */
|
|
51
|
+
export const SHARD_EXTENSION = '.ndjson.zst'
|
|
52
|
+
|
|
53
|
+
/** Everything this plugin owns on the server lives under this prefix. */
|
|
54
|
+
export const REMOTE_PREFIX = 'usage-badge/'
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* A file beside each device's `index.json`.
|
|
58
|
+
*
|
|
59
|
+
* Apache's `PROPFIND` lists the collections under `usage-badge/`, which is how a device discovers
|
|
60
|
+
* the others — but a collection holding only `index.json` may list without its `index.json` (the
|
|
61
|
+
* server is free to omit a member), and discovery is the one thing that must not depend on which of
|
|
62
|
+
* the two a reader happens to see. So each device directory also gets this small marker, created
|
|
63
|
+
* right after the collection and never rewritten: if a reader can see the directory at all, it sees
|
|
64
|
+
* the marker, and a directory without it is treated as half-created rather than as a device.
|
|
65
|
+
*/
|
|
66
|
+
export const DEVICE_MARKER = '.device'
|
|
67
|
+
|
|
68
|
+
const sha256 = (value) => createHash('sha256').update(value).digest('hex')
|
|
69
|
+
|
|
70
|
+
/** The month a day key belongs to: `2026-09-14` → `2026-09`. */
|
|
71
|
+
export const monthOf = (date) => String(date ?? '').slice(0, 7)
|
|
72
|
+
|
|
73
|
+
/** One shard's path, given a device id and a month. */
|
|
74
|
+
export const shardPath = (device, key) => `${REMOTE_PREFIX}${device}/${key}${SHARD_EXTENSION}`
|
|
75
|
+
|
|
76
|
+
/** One device's manifest path. */
|
|
77
|
+
export const manifestPath = (device) => `${REMOTE_PREFIX}${device}/index.json`
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* A device id as it may appear in a path: no slashes, no traversal, nothing exotic.
|
|
81
|
+
*
|
|
82
|
+
* `local` is refused on purpose. A device's own rows are stored with `source = 'local'` until they are
|
|
83
|
+
* exported, and `chunkRows` decides ownership by that exact value — so a device *named* `local` would
|
|
84
|
+
* make every other device mistake its rows for their own and re-publish them under their own name, after
|
|
85
|
+
* which this device's own fold could delete them. The name is a path segment and an ownership tag; it
|
|
86
|
+
* cannot be the tag that means "not from the sync".
|
|
87
|
+
*/
|
|
88
|
+
export const safeDeviceId = (device) => {
|
|
89
|
+
const id = String(device ?? '').trim()
|
|
90
|
+
if (id === LOCAL_SOURCE) throw new Error(`\`${LOCAL_SOURCE}\` is reserved for this machine's own rows and cannot be a device id`)
|
|
91
|
+
if (!/^[A-Za-z0-9._-]{1,64}$/.test(id) || id === '.' || id === '..') {
|
|
92
|
+
throw new Error(`a device id must be 1-64 of [A-Za-z0-9._-], got ${JSON.stringify(device)}`)
|
|
93
|
+
}
|
|
94
|
+
return id
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** A shard key as it may appear in a path: exactly one `YYYY-MM`. */
|
|
98
|
+
export const safeShardKey = (key) => {
|
|
99
|
+
const value = String(key ?? '').trim()
|
|
100
|
+
if (!/^\d{4}-\d{2}$/.test(value)) throw new Error(`a shard key must be YYYY-MM, got ${JSON.stringify(key)}`)
|
|
101
|
+
return value
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* One device's local rows grouped into month shards.
|
|
106
|
+
*
|
|
107
|
+
* Only `current` rows this device folded are exported (`owns` selects them; it is a parameter so a
|
|
108
|
+
* test can exercise the filter without pretending to be another device). Each row is copied out with
|
|
109
|
+
* its `sessionId` attached, because a shard has to be able to rebuild a session record on the other
|
|
110
|
+
* side.
|
|
111
|
+
*
|
|
112
|
+
* @param {object[]} records - records as `store.scan()` returns them, each with a `rows` array.
|
|
113
|
+
* @param {{owns?:string}} [options]
|
|
114
|
+
* @returns {Map<string, object[]>} shard key → rows, sorted by date then session.
|
|
115
|
+
*/
|
|
116
|
+
export function chunkRows(records, { owns = LOCAL_SOURCE } = {}) {
|
|
117
|
+
const shards = new Map()
|
|
118
|
+
for (const record of records ?? []) {
|
|
119
|
+
for (const row of record?.rows ?? []) {
|
|
120
|
+
if (rowSource(row) !== owns) continue
|
|
121
|
+
if (rowKind(row) === FALLBACK) continue
|
|
122
|
+
const key = monthOf(row.date)
|
|
123
|
+
if (!/^\d{4}-\d{2}$/.test(key)) continue
|
|
124
|
+
let list = shards.get(key)
|
|
125
|
+
if (!list) {
|
|
126
|
+
list = []
|
|
127
|
+
shards.set(key, list)
|
|
128
|
+
}
|
|
129
|
+
list.push({
|
|
130
|
+
date: row.date,
|
|
131
|
+
source: rowSource(row),
|
|
132
|
+
kind: CURRENT,
|
|
133
|
+
seq: Number(row.seq) || 0,
|
|
134
|
+
fileSize: Number(row.fileSize) || 0,
|
|
135
|
+
fileMtimeMs: Number(row.fileMtimeMs) || 0,
|
|
136
|
+
at: Number(row.at) || 0,
|
|
137
|
+
sessionId: String(record.sessionId ?? ''),
|
|
138
|
+
path: String(record.path ?? ''),
|
|
139
|
+
routes: row.routes ?? {},
|
|
140
|
+
})
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
for (const list of shards.values()) {
|
|
144
|
+
list.sort((a, b) => (a.date === b.date ? (a.sessionId < b.sessionId ? -1 : a.sessionId > b.sessionId ? 1 : 0) : a.date < b.date ? -1 : 1))
|
|
145
|
+
}
|
|
146
|
+
return shards
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* One shard's bytes: a header line, then one JSON row per line, compressed as a whole.
|
|
151
|
+
*
|
|
152
|
+
* Two properties are load-bearing:
|
|
153
|
+
*
|
|
154
|
+
* - **the bytes are deterministic.** Nothing about *when* the shard was written goes into them, so
|
|
155
|
+
* the upload plan can decide "this month did not change" by comparing a content hash. If the
|
|
156
|
+
* timestamp were part of the payload, every re-plan would look like a new revision and every sync
|
|
157
|
+
* would re-upload everything. When a month was written is manifest metadata, not shard data.
|
|
158
|
+
* - **a row's `source` is rewritten to the publishing device.** A row stored here says
|
|
159
|
+
* `source = local`, meaning "this device folded it"; on the receiving side the same bytes must
|
|
160
|
+
* mean "device `<id>` published it". Carrying `local` across would make the receiver treat
|
|
161
|
+
* another machine's history as **its own** — it would then publish it onward under its own name,
|
|
162
|
+
* and a later fold here could delete rows it does not own.
|
|
163
|
+
*/
|
|
164
|
+
export function encodeShard(rows, { device }) {
|
|
165
|
+
const id = safeDeviceId(device)
|
|
166
|
+
const header = JSON.stringify({ v: SHARD_VERSION, device: id })
|
|
167
|
+
const body = [
|
|
168
|
+
header,
|
|
169
|
+
...rows.map((row) => JSON.stringify({ ...row, source: rowSource(row) === LOCAL_SOURCE ? id : rowSource(row) })),
|
|
170
|
+
].join('\n')
|
|
171
|
+
const compressed = zstdCompressSync(Buffer.from(body, 'utf8'))
|
|
172
|
+
return { body: compressed, sha256: sha256(compressed), rows: rows.length }
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Decode one shard.
|
|
177
|
+
*
|
|
178
|
+
* A shard whose version is not this build's is refused **by name** rather than parsed leniently: a
|
|
179
|
+
* half-understood shard would be merged as if it were understood, and the result would be wrong
|
|
180
|
+
* numbers that look right.
|
|
181
|
+
*
|
|
182
|
+
* @param {Buffer|Uint8Array} buffer
|
|
183
|
+
* @returns {{version:number, device:string, exportedAt:number, rows:object[]}}
|
|
184
|
+
*/
|
|
185
|
+
export function decodeShard(buffer) {
|
|
186
|
+
let text
|
|
187
|
+
try {
|
|
188
|
+
text = zstdDecompressSync(buffer).toString('utf8')
|
|
189
|
+
} catch (error) {
|
|
190
|
+
throw new Error(`a shard could not be decompressed: ${error?.message ?? error}`)
|
|
191
|
+
}
|
|
192
|
+
const lines = text.split('\n')
|
|
193
|
+
if (!lines[0]) throw new Error('a shard is empty')
|
|
194
|
+
let header
|
|
195
|
+
try {
|
|
196
|
+
header = JSON.parse(lines[0])
|
|
197
|
+
} catch (error) {
|
|
198
|
+
throw new Error(`a shard header is not JSON: ${error?.message ?? error}`)
|
|
199
|
+
}
|
|
200
|
+
if (header?.v !== SHARD_VERSION) {
|
|
201
|
+
throw new Error(`a shard is version ${String(header?.v)}, but this build writes version ${SHARD_VERSION}`)
|
|
202
|
+
}
|
|
203
|
+
const rows = []
|
|
204
|
+
let malformed = 0
|
|
205
|
+
for (let i = 1; i < lines.length; i++) {
|
|
206
|
+
if (!lines[i]) continue
|
|
207
|
+
let row
|
|
208
|
+
try {
|
|
209
|
+
row = JSON.parse(lines[i])
|
|
210
|
+
} catch {
|
|
211
|
+
malformed++
|
|
212
|
+
continue
|
|
213
|
+
}
|
|
214
|
+
if (typeof row?.date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(row.date)) {
|
|
215
|
+
malformed++
|
|
216
|
+
continue
|
|
217
|
+
}
|
|
218
|
+
rows.push({
|
|
219
|
+
date: row.date,
|
|
220
|
+
source: String(row.source ?? header.device ?? LOCAL_SOURCE),
|
|
221
|
+
kind: row.kind === FALLBACK ? FALLBACK : CURRENT,
|
|
222
|
+
seq: Number(row.seq) || 0,
|
|
223
|
+
fileSize: Number(row.fileSize) || 0,
|
|
224
|
+
fileMtimeMs: Number(row.fileMtimeMs) || 0,
|
|
225
|
+
at: Number(row.at) || 0,
|
|
226
|
+
sessionId: String(row.sessionId ?? ''),
|
|
227
|
+
routes: row.routes && typeof row.routes === 'object' ? row.routes : {},
|
|
228
|
+
})
|
|
229
|
+
}
|
|
230
|
+
return { version: header.v, device: String(header.device ?? ''), exportedAt: Number(header.at) || 0, rows, malformed }
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** One chunk's manifest entry, describing the bytes that were actually written. */
|
|
234
|
+
export function chunkEntry({ device, key, body, rows, at }) {
|
|
235
|
+
const dates = rows.map((row) => row.date).sort()
|
|
236
|
+
return {
|
|
237
|
+
key,
|
|
238
|
+
path: shardPath(device, key),
|
|
239
|
+
from: dates[0] ?? '',
|
|
240
|
+
to: dates[dates.length - 1] ?? '',
|
|
241
|
+
bytes: body.length,
|
|
242
|
+
sha256: sha256(body),
|
|
243
|
+
rows: rows.length,
|
|
244
|
+
at,
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** A device's manifest: what this device has published. */
|
|
249
|
+
export function buildManifest({ device, label = '', chunks = {}, at = Date.now(), instanceId = '' }) {
|
|
250
|
+
// `instanceId` is written so that the *other* installations can tell whether the name in this
|
|
251
|
+
// directory is theirs. Two machines that both call themselves `office` would otherwise each overwrite
|
|
252
|
+
// the other's month — the shard path is the same — and the rows would be gone with no warning at all.
|
|
253
|
+
return { v: MANIFEST_VERSION, device, label, generatedAt: at, instanceId, chunks }
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** Parse a manifest, refusing a version this build does not write and a device id it cannot use. */
|
|
257
|
+
export function parseManifest(text) {
|
|
258
|
+
let raw
|
|
259
|
+
try {
|
|
260
|
+
raw = typeof text === 'string' || Buffer.isBuffer(text) ? JSON.parse(text.toString('utf8')) : text
|
|
261
|
+
} catch (error) {
|
|
262
|
+
throw new Error(`the manifest is not JSON: ${error?.message ?? error}`)
|
|
263
|
+
}
|
|
264
|
+
if (raw?.v !== MANIFEST_VERSION) {
|
|
265
|
+
throw new Error(`the manifest is version ${String(raw?.v)}, but this build writes version ${MANIFEST_VERSION}`)
|
|
266
|
+
}
|
|
267
|
+
const device = safeDeviceId(raw.device)
|
|
268
|
+
const chunks = {}
|
|
269
|
+
for (const [key, entry] of Object.entries(raw.chunks ?? {})) {
|
|
270
|
+
if (!entry || typeof entry !== 'object' || typeof entry.sha256 !== 'string') continue
|
|
271
|
+
chunks[key] = {
|
|
272
|
+
key,
|
|
273
|
+
path: String(entry.path ?? shardPath(device, key)),
|
|
274
|
+
from: String(entry.from ?? ''),
|
|
275
|
+
to: String(entry.to ?? ''),
|
|
276
|
+
bytes: Number(entry.bytes) || 0,
|
|
277
|
+
sha256: entry.sha256,
|
|
278
|
+
rows: Number(entry.rows) || 0,
|
|
279
|
+
at: Number(entry.at) || 0,
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
return { version: raw.v, device, label: String(raw.label ?? ''), generatedAt: Number(raw.generatedAt) || 0, instanceId: String(raw.instanceId ?? ''), chunks }
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Decide what to upload: the shards whose bytes differ from what this device last published.
|
|
287
|
+
*
|
|
288
|
+
* The comparison is by **content hash**, not by date or mtime: the sync is incremental and has to
|
|
289
|
+
* skip a month that has not changed, and a re-fold that reproduces identical bytes must not become
|
|
290
|
+
* an upload just because it happened.
|
|
291
|
+
*
|
|
292
|
+
* @param {Map<string, object[]>} shards - as {@link chunkRows} returns.
|
|
293
|
+
* @param {object} [previous] - the manifest this device published last time, or `{}`.
|
|
294
|
+
* @returns {{pending: Array<{key:string, body:Buffer, entry:object, rows:number}>, manifest:object, unchanged:number}}
|
|
295
|
+
*/
|
|
296
|
+
export function planUpload(shards, previous = {}, { device, label = '', at = Date.now(), instanceId = '' } = {}) {
|
|
297
|
+
const safeDevice = safeDeviceId(device)
|
|
298
|
+
const chunks = { ...(previous?.chunks ?? {}) }
|
|
299
|
+
const pending = []
|
|
300
|
+
let unchanged = 0
|
|
301
|
+
for (const [key, rows] of [...(shards ?? new Map())].sort(([a], [b]) => (a < b ? -1 : 1))) {
|
|
302
|
+
const { body } = encodeShard(rows, { device: safeDevice })
|
|
303
|
+
const hash = sha256(body)
|
|
304
|
+
const known = chunks[key]
|
|
305
|
+
if (known?.sha256 === hash) {
|
|
306
|
+
// Left exactly as published, timestamp included: re-stamping it would say the month changed.
|
|
307
|
+
unchanged++
|
|
308
|
+
continue
|
|
309
|
+
}
|
|
310
|
+
const entry = chunkEntry({ device: safeDevice, key, body, rows, at })
|
|
311
|
+
chunks[key] = entry
|
|
312
|
+
pending.push({ key, body, entry, rows: rows.length })
|
|
313
|
+
}
|
|
314
|
+
return { pending, unchanged, manifest: buildManifest({ device: safeDevice, label, chunks, at, instanceId }) }
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* The shards a reader should fetch.
|
|
319
|
+
*
|
|
320
|
+
* `windowYears` bounds the **first** pull: a decade of history is real, but downloading all of it
|
|
321
|
+
* before the panel shows anything would be the worst first impression this feature could make. The
|
|
322
|
+
* bound applies to fetching, never to keeping — shards already held stay held, and each device's
|
|
323
|
+
* manifest still lists everything it has.
|
|
324
|
+
*
|
|
325
|
+
* @param {{manifests:object[], held?:object, now?:number, windowYears?:number, except?:string}} options
|
|
326
|
+
* `held` maps `<device>/<key>` to the sha256 this reader already applied. `except` is the reader's
|
|
327
|
+
* own device id: its own manifest lists what it published, which is already in its store.
|
|
328
|
+
*/
|
|
329
|
+
export function planDownload({ manifests, held = {}, now = Date.now(), windowYears = 5, except = '' }) {
|
|
330
|
+
const cutoff = `${new Date(now).getFullYear() - Math.max(0, windowYears) + 1}-01`
|
|
331
|
+
const wanted = []
|
|
332
|
+
for (const manifest of manifests ?? []) {
|
|
333
|
+
if (except && manifest?.device === except) continue
|
|
334
|
+
for (const [key, entry] of Object.entries(manifest?.chunks ?? {})) {
|
|
335
|
+
if (!entry?.path || !entry?.sha256) continue
|
|
336
|
+
if (windowYears > 0 && key < cutoff) continue
|
|
337
|
+
if (held[`${manifest.device}/${key}`] === entry.sha256) continue
|
|
338
|
+
wanted.push({ device: manifest.device, key, entry })
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
wanted.sort((a, b) => (a.key === b.key ? (a.device < b.device ? -1 : 1) : a.key < b.key ? -1 : 1))
|
|
342
|
+
return wanted
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* Turn decoded shard rows into the records the store's import takes.
|
|
347
|
+
*
|
|
348
|
+
* Grouped by `(source, sessionId)` because that is the store's write unit, and every row keeps the
|
|
349
|
+
* stamp it was written with: the receiving store compares those stamps to decide whether the
|
|
350
|
+
* incoming copy is worth storing at all. The cursor fields on the record are the best stamp among
|
|
351
|
+
* its rows, so a caller that only needs "the newest log version this device knows" can read them.
|
|
352
|
+
*/
|
|
353
|
+
export function rowsToRecords(rows) {
|
|
354
|
+
const bySource = new Map()
|
|
355
|
+
for (const row of rows ?? []) {
|
|
356
|
+
if (!row?.sessionId) continue
|
|
357
|
+
const key = `${row.source}\u0000${row.sessionId}`
|
|
358
|
+
let record = bySource.get(key)
|
|
359
|
+
if (!record) {
|
|
360
|
+
record = { sessionId: row.sessionId, path: '', source: row.source, rows: [] }
|
|
361
|
+
bySource.set(key, record)
|
|
362
|
+
}
|
|
363
|
+
record.rows.push(row)
|
|
364
|
+
}
|
|
365
|
+
for (const record of bySource.values()) {
|
|
366
|
+
const best = record.rows.reduce((top, row) => (top === null || row.seq > top.seq || (row.seq === top.seq && row.fileSize > top.fileSize) ? row : top), null)
|
|
367
|
+
record.seq = best?.seq ?? -1
|
|
368
|
+
record.fileSize = best?.fileSize ?? 0
|
|
369
|
+
record.fileMtimeMs = best?.fileMtimeMs ?? 0
|
|
370
|
+
}
|
|
371
|
+
return [...bySource.values()]
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/** A short, display-safe prefix of a hash, for logs and the panel. */
|
|
375
|
+
export const shortHash = (hash) => String(hash ?? '').slice(0, 12)
|
|
376
|
+
|
|
377
|
+
// ── the price table: one shared file, and one copy per device ────────────────
|
|
378
|
+
|
|
379
|
+
/** The shared price table, which every device reads and the last writer wins. */
|
|
380
|
+
export const PRICING_FILE = `${REMOTE_PREFIX}pricing.json`
|
|
381
|
+
|
|
382
|
+
/** The price table this device wrote, kept so a conflict can always be resolved by hand. */
|
|
383
|
+
export const pricingDevicePath = (device) => `${REMOTE_PREFIX}${safeDeviceId(device)}/pricing-${safeDeviceId(device)}.json`
|
|
384
|
+
|
|
385
|
+
/** Byte-exact comparison, since the plan is about *whether* the file changed, not how. */
|
|
386
|
+
const sameBytes = (a, b) => (a === null || b === null ? a === b : Buffer.compare(Buffer.from(a), Buffer.from(b)) === 0)
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* Decide what to do about the shared price table.
|
|
390
|
+
*
|
|
391
|
+
* The price table is the one thing here that a human writes by hand, so it cannot be merged by a
|
|
392
|
+
* rule — two people editing the same table is a decision only a person can make. What this can do is
|
|
393
|
+
* tell the three cases apart without ever losing a version:
|
|
394
|
+
*
|
|
395
|
+
* - **nothing changed** (`remote === baseline`) → do nothing, whichever device wrote last;
|
|
396
|
+
* - **only the remote moved** → take it;
|
|
397
|
+
* - **only we moved** → publish ours;
|
|
398
|
+
* - **both moved, differently** → a conflict: the remote is kept beside ours as
|
|
399
|
+
* `.conflict-pricing-<device>-<at>.json` before either wins, and the later `at` wins. Losing a
|
|
400
|
+
* price row silently is exactly the outcome this whole design is built to avoid.
|
|
401
|
+
*
|
|
402
|
+
* @param {{localBytes?:Buffer|null, remoteBytes?:Buffer|null, baseline?:string|null,
|
|
403
|
+
* localAt?:number, remoteAt?:number, device?:string, at?:number}} options
|
|
404
|
+
* @returns {{action:'none'|'pull'|'push'|'conflict', winner:'local'|'remote'|null, conflictFile:string|null}}
|
|
405
|
+
*/
|
|
406
|
+
export function planPricing({ localBytes = null, remoteBytes = null, baseline = null, localAt = 0, remoteAt = 0, device = '', at = 0 } = {}) {
|
|
407
|
+
const localHash = localBytes === null ? null : sha256(localBytes)
|
|
408
|
+
const remoteHash = remoteBytes === null ? null : sha256(remoteBytes)
|
|
409
|
+
// A missing local file means "we have not published anything yet", not "we changed something":
|
|
410
|
+
// otherwise a fresh install calls the server's very first table a conflict. An *empty* file is a
|
|
411
|
+
// different thing — it exists, and it is not what the server has.
|
|
412
|
+
const localChanged = localHash === null ? baseline !== null : localHash !== baseline
|
|
413
|
+
const remoteChanged = remoteHash !== null && remoteHash !== baseline
|
|
414
|
+
|
|
415
|
+
// Nothing to take and nothing to publish: either both sides agree, or the file we published is
|
|
416
|
+
// still the one on the server.
|
|
417
|
+
if (!localChanged && !remoteChanged) return { action: 'none', winner: null, conflictFile: null }
|
|
418
|
+
if (!localChanged && remoteChanged) return { action: 'pull', winner: 'remote', conflictFile: null }
|
|
419
|
+
if (localChanged && !remoteChanged) return { action: 'push', winner: 'local', conflictFile: null }
|
|
420
|
+
if (sameBytes(localBytes, remoteBytes)) return { action: 'none', winner: 'remote', conflictFile: null }
|
|
421
|
+
|
|
422
|
+
// Both sides changed, and they disagree. Keep the one that would have been lost.
|
|
423
|
+
const winner = remoteAt >= localAt ? 'remote' : 'local'
|
|
424
|
+
const loserPath = remoteAt >= localAt ? localBytes : remoteBytes
|
|
425
|
+
const conflictFile = device && loserPath !== null ? `.conflict-pricing-${safeDeviceId(device)}-${at || Date.now()}.json` : null
|
|
426
|
+
return { action: 'conflict', winner, conflictFile }
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// ── the data directory is owned by one process at a time ─────────────────────
|
|
430
|
+
|
|
431
|
+
/** The lock file a host holds while it may write this data directory. */
|
|
432
|
+
export const LOCK_FILE = 'host.lock'
|
|
433
|
+
|
|
434
|
+
/**
|
|
435
|
+
* Whether the process that wrote a lock file is still alive.
|
|
436
|
+
*
|
|
437
|
+
* `process.kill(pid, 0)` is the portable probe: it sends no signal, and its failure modes separate
|
|
438
|
+
* "no such process" (ESRCH — the lock is stale, take it) from "not allowed to signal it" (EPERM —
|
|
439
|
+
* there *is* a process, and on Windows that is the normal answer for a process owned by another
|
|
440
|
+
* session). Anything else is treated as "cannot tell", which means **not** taking the lock: a
|
|
441
|
+
* second writer is worse than a read-only instance.
|
|
442
|
+
*/
|
|
443
|
+
export function processLooksAlive(pid) {
|
|
444
|
+
const value = Number(pid)
|
|
445
|
+
if (!Number.isInteger(value) || value <= 0) return false
|
|
446
|
+
if (value === process.pid) return true
|
|
447
|
+
try {
|
|
448
|
+
process.kill(value, 0)
|
|
449
|
+
return true
|
|
450
|
+
} catch (error) {
|
|
451
|
+
const code = error?.code
|
|
452
|
+
if (code === 'ESRCH') return false
|
|
453
|
+
if (code === 'EPERM') return true
|
|
454
|
+
return true
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* Write the lock, and read back what a previous holder left.
|
|
460
|
+
*
|
|
461
|
+
* `process.pid` alone is not enough: pid reuse after a crash would make a live-looking lock out of a
|
|
462
|
+
* dead process. Comparing the recorded process start time catches that, and the comparison is
|
|
463
|
+
* deliberately lax — a mismatch in the *started* field only downgrades the lock to "someone else
|
|
464
|
+
* holds it", which is the safe direction.
|
|
465
|
+
*/
|
|
466
|
+
export function readLock(dir) {
|
|
467
|
+
const path = join(dir, LOCK_FILE)
|
|
468
|
+
try {
|
|
469
|
+
const raw = JSON.parse(readFileSync(path, 'utf8'))
|
|
470
|
+
return {
|
|
471
|
+
pid: Number(raw?.pid) || 0,
|
|
472
|
+
startedAt: Number(raw?.startedAt) || 0,
|
|
473
|
+
instanceId: String(raw?.instanceId ?? ''),
|
|
474
|
+
hostname: String(raw?.hostname ?? ''),
|
|
475
|
+
at: Number(raw?.at) || 0,
|
|
476
|
+
}
|
|
477
|
+
} catch {
|
|
478
|
+
return null
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/** Whether a lock belongs to a process that is no longer running. */
|
|
483
|
+
export const lockIsStale = (lock) => lock === null || !processLooksAlive(lock.pid)
|
|
484
|
+
|
|
485
|
+
/** Just this process's own start, in epoch ms. */
|
|
486
|
+
const PROCESS_STARTED_AT = Date.now() - Math.round(process.uptime() * 1000)
|
|
487
|
+
|
|
488
|
+
/**
|
|
489
|
+
* Try to take ownership of a data directory.
|
|
490
|
+
*
|
|
491
|
+
* Ownership decides who may *write*: the cache, the fold, and the sync all need one writer, and the
|
|
492
|
+
* sync needs it most — two processes sharing a `deviceId` would each believe the same remote shard is
|
|
493
|
+
* theirs to overwrite. The caller that fails here is not broken and does not refuse to start: it
|
|
494
|
+
* uses the cache read-only and skips syncing, which is what the design promises for this case.
|
|
495
|
+
*
|
|
496
|
+
* @returns {{acquired:boolean, owner?:object, reason?:string}}
|
|
497
|
+
*/
|
|
498
|
+
export function acquireLock(dir, { instanceId, log = () => {} } = {}) {
|
|
499
|
+
try {
|
|
500
|
+
// Creating the data directory is part of taking the lock, and it can fail for reasons that have
|
|
501
|
+
// nothing to do with a lock: a read-only home, or a `storages` that turned out to be a file. Those are
|
|
502
|
+
// reported like any other "cannot write here" — an exception escaping here reached the panel as a 500
|
|
503
|
+
// with no explanation.
|
|
504
|
+
mkdirSync(dir, { recursive: true })
|
|
505
|
+
} catch (error) {
|
|
506
|
+
return { acquired: false, owner: null, reason: `the data directory cannot be created: ${error?.message ?? error}` }
|
|
507
|
+
}
|
|
508
|
+
const path = join(dir, LOCK_FILE)
|
|
509
|
+
const existing = readLock(dir)
|
|
510
|
+
if (existing && !lockIsStale(existing)) {
|
|
511
|
+
return {
|
|
512
|
+
acquired: false,
|
|
513
|
+
owner: existing,
|
|
514
|
+
reason: `this data directory is open by another DSH instance (pid ${existing.pid} on ${existing.hostname || 'this host'})`,
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
if (existing) log(`taking over a stale ${LOCK_FILE} left by pid ${existing.pid}`)
|
|
518
|
+
const body = { pid: process.pid, startedAt: PROCESS_STARTED_AT, instanceId, hostname: hostname(), at: Date.now() }
|
|
519
|
+
try {
|
|
520
|
+
writeFileSync(path, JSON.stringify(body))
|
|
521
|
+
} catch (error) {
|
|
522
|
+
return { acquired: false, owner: null, reason: `could not write ${LOCK_FILE}: ${error?.message ?? error}` }
|
|
523
|
+
}
|
|
524
|
+
return { acquired: true, owner: body }
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/** Release the lock, but only if this process is the one holding it. */
|
|
528
|
+
export function releaseLock(dir) {
|
|
529
|
+
const existing = readLock(dir)
|
|
530
|
+
if (!existing || existing.pid !== process.pid) return false
|
|
531
|
+
try {
|
|
532
|
+
unlinkSync(join(dir, LOCK_FILE))
|
|
533
|
+
return true
|
|
534
|
+
} catch {
|
|
535
|
+
return false
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// ── sync.json: what the user configured, and what the last run learned ───────
|
|
540
|
+
|
|
541
|
+
/** The file holding the sync configuration and the last run's state. */
|
|
542
|
+
export const SYNC_FILE = 'sync.json'
|
|
543
|
+
|
|
544
|
+
/** The defaults a sync configuration starts from. */
|
|
545
|
+
export const defaultSyncConfig = () => ({
|
|
546
|
+
enabled: false,
|
|
547
|
+
url: '',
|
|
548
|
+
username: '',
|
|
549
|
+
realm: '',
|
|
550
|
+
ha1: '',
|
|
551
|
+
allowInsecureTls: false,
|
|
552
|
+
deviceId: '',
|
|
553
|
+
deviceLabel: hostname(),
|
|
554
|
+
instanceId: '',
|
|
555
|
+
windowYears: 5,
|
|
556
|
+
})
|
|
557
|
+
|
|
558
|
+
/** Read `sync.json`, or a default document when it is missing or unreadable. */
|
|
559
|
+
export function readSyncDocument(dir, log = () => {}) {
|
|
560
|
+
const path = join(dir, SYNC_FILE)
|
|
561
|
+
try {
|
|
562
|
+
const raw = JSON.parse(readFileSync(path, 'utf8'))
|
|
563
|
+
if (!raw || typeof raw !== 'object') return { ...defaultSyncConfig(), state: {}, known: {} }
|
|
564
|
+
return { ...defaultSyncConfig(), ...raw }
|
|
565
|
+
} catch (error) {
|
|
566
|
+
if (error?.code !== 'ENOENT') log(`${SYNC_FILE} could not be read (${error?.message ?? error}); using the defaults`)
|
|
567
|
+
return { ...defaultSyncConfig(), state: {}, known: {} }
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
/** Write `sync.json` atomically, so a crash cannot leave a half-written configuration. */
|
|
572
|
+
export function writeSyncDocument(dir, document) {
|
|
573
|
+
mkdirSync(dir, { recursive: true })
|
|
574
|
+
const path = join(dir, SYNC_FILE)
|
|
575
|
+
const temp = `${path}.tmp-${process.pid}`
|
|
576
|
+
writeFileSync(temp, JSON.stringify(document, null, 2))
|
|
577
|
+
try {
|
|
578
|
+
renameSync(temp, path)
|
|
579
|
+
} catch {
|
|
580
|
+
try {
|
|
581
|
+
unlinkSync(path)
|
|
582
|
+
} catch {
|
|
583
|
+
// absent is fine
|
|
584
|
+
}
|
|
585
|
+
renameSync(temp, path)
|
|
586
|
+
}
|
|
587
|
+
return path
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
// ── the orchestration: discover, pull, push ──────────────────────────────────
|
|
591
|
+
|
|
592
|
+
/**
|
|
593
|
+
* One device's view of the whole remote, kept between runs.
|
|
594
|
+
*
|
|
595
|
+
* `known.shards` maps `<device>/<key>` to the sha256 already applied, which is what makes a second
|
|
596
|
+
* run cheap: it is also the record that proves a shard was read rather than merely listed. It is
|
|
597
|
+
* rebuilt from the remote on every run, so losing it costs a re-download and nothing else.
|
|
598
|
+
*/
|
|
599
|
+
export const defaultSyncState = () => ({
|
|
600
|
+
lastSyncAt: 0,
|
|
601
|
+
lastResult: 'never',
|
|
602
|
+
message: '',
|
|
603
|
+
held: {},
|
|
604
|
+
published: {},
|
|
605
|
+
})
|
|
606
|
+
|
|
607
|
+
/**
|
|
608
|
+
* The remote orchestration.
|
|
609
|
+
*
|
|
610
|
+
* It owns the order of operations, because the order is the correctness argument: take the lock,
|
|
611
|
+
* **pull before pushing** (so a shard this device is about to overwrite is already accounted for),
|
|
612
|
+
* import into the store, publish only what changed, and write the manifest last — a manifest that
|
|
613
|
+
* names a shard which is not there yet would send every other device after a 404.
|
|
614
|
+
*/
|
|
615
|
+
export class SyncEngine {
|
|
616
|
+
/**
|
|
617
|
+
* @param {{dataDir:string, store?:object, log?:Function, client?:object, now?:Function,
|
|
618
|
+
* config?:object, state?:object, instanceId?:string}} options
|
|
619
|
+
* `store` may be null: a sync without a store still publishes and reads manifests, which is what
|
|
620
|
+
* the first step's live check exercises.
|
|
621
|
+
*/
|
|
622
|
+
constructor({ dataDir, store = null, log = () => {}, client = null, now = Date.now, config = null, state = null, instanceId = '' }) {
|
|
623
|
+
this.dataDir = dataDir
|
|
624
|
+
this.store = store
|
|
625
|
+
this.log = log
|
|
626
|
+
this.now = now
|
|
627
|
+
const document = config ? { ...defaultSyncConfig(), ...config } : readSyncDocument(dataDir, log)
|
|
628
|
+
this.config = document
|
|
629
|
+
this.state = state ?? { ...defaultSyncState(), ...(document.state ?? {}) }
|
|
630
|
+
this.instanceId = instanceId || document.instanceId || `${hostname()}-${process.pid}`
|
|
631
|
+
this.client = client
|
|
632
|
+
this.lock = null
|
|
633
|
+
this.ownsLock = false
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/**
|
|
637
|
+
* Give up ownership of the data directory.
|
|
638
|
+
*
|
|
639
|
+
* Called on unload by the host half, and by anything embedding this engine for one run. It only
|
|
640
|
+
* removes a lock this engine took: a lock handed in by the caller belongs to the caller.
|
|
641
|
+
*/
|
|
642
|
+
release() {
|
|
643
|
+
if (!this.ownsLock) return false
|
|
644
|
+
const released = releaseLock(this.dataDir)
|
|
645
|
+
this.ownsLock = false
|
|
646
|
+
return released
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
get deviceId() {
|
|
650
|
+
return this.config.deviceId
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
/** Persist the configuration (and the latest state) to `sync.json`. */
|
|
654
|
+
save({ config = {}, state = {} } = {}) {
|
|
655
|
+
this.config = { ...this.config, ...config, instanceId: this.instanceId }
|
|
656
|
+
this.state = { ...this.state, ...state }
|
|
657
|
+
writeSyncDocument(this.dataDir, { ...this.config, state: this.state })
|
|
658
|
+
return this.config
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/** A WebDAV client for the configured server, or a clear failure. */
|
|
662
|
+
connect() {
|
|
663
|
+
if (this.client) return this.client
|
|
664
|
+
const { url, username, ha1 } = this.config
|
|
665
|
+
if (!url) throw new Error('no WebDAV URL is configured')
|
|
666
|
+
if (!username) throw new Error('no WebDAV username is configured')
|
|
667
|
+
if (!ha1) throw new Error('no WebDAV credential is configured')
|
|
668
|
+
return new WebDavClient({
|
|
669
|
+
baseUrl: url,
|
|
670
|
+
username,
|
|
671
|
+
// The stored secret is the digest HA1 — `MD5(user:realm:pass)`, exactly what the server keeps in
|
|
672
|
+
// its own password file. Passing it as `passwordHash` means the plaintext never has to be
|
|
673
|
+
// written down here, and the response is still the one the server expects.
|
|
674
|
+
passwordHash: ha1,
|
|
675
|
+
allowInsecureTls: Boolean(this.config.allowInsecureTls),
|
|
676
|
+
})
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/**
|
|
680
|
+
* The device directories under `usage-badge/`.
|
|
681
|
+
*
|
|
682
|
+
* It lists the `usage-badge/` collection itself rather than the root: the root's children include
|
|
683
|
+
* the collection's own name, and treating that as a device would send this sync after
|
|
684
|
+
* `usage-badge/usage-badge/index.json`. A collection that does not exist yet means an empty list —
|
|
685
|
+
* the first device to sync creates it.
|
|
686
|
+
*/
|
|
687
|
+
async discover(client) {
|
|
688
|
+
let entries
|
|
689
|
+
try {
|
|
690
|
+
entries = await client.propfind(REMOTE_PREFIX)
|
|
691
|
+
} catch (error) {
|
|
692
|
+
// `PROPFIND` answers 404 for a path that has no collection yet; that is "nothing here", not a
|
|
693
|
+
// failure to talk to the server.
|
|
694
|
+
if (/answered 404/.test(String(error?.message ?? ''))) return []
|
|
695
|
+
throw new Error(`could not list ${REMOTE_PREFIX}: ${error?.message ?? error}`)
|
|
696
|
+
}
|
|
697
|
+
const devices = []
|
|
698
|
+
for (const entry of entries) {
|
|
699
|
+
if (!entry.collection) continue
|
|
700
|
+
const name = String(entry.path ?? entry.name).replace(/\/+$/, '').split('/').pop() ?? ''
|
|
701
|
+
if (!/^[A-Za-z0-9._-]{1,64}$/.test(name) || name === '.' || name === '..') continue
|
|
702
|
+
devices.push(name)
|
|
703
|
+
}
|
|
704
|
+
return devices
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
/** Read one device's manifest, or null when it has none yet. */
|
|
708
|
+
async readManifest(client, device) {
|
|
709
|
+
const body = await client.get(manifestPath(device))
|
|
710
|
+
if (body === null) return null
|
|
711
|
+
try {
|
|
712
|
+
return parseManifest(body)
|
|
713
|
+
} catch (error) {
|
|
714
|
+
// A manifest that cannot be parsed is reported and skipped: refusing the whole sync over one
|
|
715
|
+
// unreadable device would stop the devices that are fine.
|
|
716
|
+
this.log(`manifest of ${device} could not be read: ${error?.message ?? error}`)
|
|
717
|
+
return null
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
/** Everything the remote knows, as manifests. */
|
|
722
|
+
async readManifests(client) {
|
|
723
|
+
const devices = await this.discover(client)
|
|
724
|
+
const manifests = []
|
|
725
|
+
for (const device of devices) {
|
|
726
|
+
const manifest = await this.readManifest(client, device)
|
|
727
|
+
if (manifest) manifests.push(manifest)
|
|
728
|
+
}
|
|
729
|
+
return manifests
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
/**
|
|
733
|
+
* Pull: fetch the shards this device does not already hold, and store their rows.
|
|
734
|
+
*
|
|
735
|
+
* @returns {{fetched:number, rows:number, failed:Array<{path:string, error:string}>}}
|
|
736
|
+
*/
|
|
737
|
+
async pull(client, manifests) {
|
|
738
|
+
const wanted = planDownload({
|
|
739
|
+
manifests,
|
|
740
|
+
held: this.state.held ?? {},
|
|
741
|
+
now: this.now(),
|
|
742
|
+
windowYears: Number(this.config.windowYears) || 0,
|
|
743
|
+
// This device's own manifest is not something to fetch: it lists the shards this device
|
|
744
|
+
// published, and its rows are already in this store under their local source. Reading them back
|
|
745
|
+
// would re-attribute them to this device's id and import duplicates of what is already here.
|
|
746
|
+
//
|
|
747
|
+
// Unless that directory is **someone else's**, which is the name clash: the shards in it are then a
|
|
748
|
+
// history this machine does not have, and it is exactly the history a reinstalled machine came
|
|
749
|
+
// back for. Importing them is safe — the merge key is the session, not the device — and the
|
|
750
|
+
// alternative is an empty panel until the user resolves a naming question they may not understand.
|
|
751
|
+
//
|
|
752
|
+
// The decision comes from **this run's** finding, never from the persisted `state.collision`: a
|
|
753
|
+
// refusal can stay recorded for months, and reading one's own directory because of a stale note
|
|
754
|
+
// means re-fetching one's own shards — including the half-written one that caused the refusal.
|
|
755
|
+
except: this.pullOwnDirectory ? '' : this.deviceId,
|
|
756
|
+
})
|
|
757
|
+
let rows = 0
|
|
758
|
+
let fetched = 0
|
|
759
|
+
const failed = []
|
|
760
|
+
for (const item of wanted) {
|
|
761
|
+
let body
|
|
762
|
+
try {
|
|
763
|
+
body = await client.get(item.entry.path)
|
|
764
|
+
} catch (error) {
|
|
765
|
+
failed.push({ path: item.entry.path, error: String(error?.message ?? error) })
|
|
766
|
+
continue
|
|
767
|
+
}
|
|
768
|
+
if (body === null) {
|
|
769
|
+
failed.push({ path: item.entry.path, error: 'the manifest lists a shard the server does not have' })
|
|
770
|
+
continue
|
|
771
|
+
}
|
|
772
|
+
let decoded
|
|
773
|
+
try {
|
|
774
|
+
decoded = decodeShard(body)
|
|
775
|
+
} catch (error) {
|
|
776
|
+
failed.push({ path: item.entry.path, error: String(error?.message ?? error) })
|
|
777
|
+
continue
|
|
778
|
+
}
|
|
779
|
+
// A shard whose hash does not match its manifest is not stored: the store keeps whatever is
|
|
780
|
+
// newer, so importing a truncated month could not be undone by re-importing the good one.
|
|
781
|
+
if (item.entry.sha256 && item.entry.sha256 !== sha256(body)) {
|
|
782
|
+
failed.push({ path: item.entry.path, error: 'the shard does not match the hash its manifest advertises' })
|
|
783
|
+
continue
|
|
784
|
+
}
|
|
785
|
+
if (this.store) {
|
|
786
|
+
this.store.intakeRows(decoded.rows)
|
|
787
|
+
// `held` is only advanced once the rows are in the store: if the import throws, the shard
|
|
788
|
+
// must be fetched again rather than remembered as applied.
|
|
789
|
+
this.state.held = { ...(this.state.held ?? {}), [`${item.device}/${item.key}`]: item.entry.sha256 }
|
|
790
|
+
}
|
|
791
|
+
rows += decoded.rows.length
|
|
792
|
+
fetched++
|
|
793
|
+
}
|
|
794
|
+
return { fetched, rows, failed }
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
/**
|
|
798
|
+
* Push: publish this device's changed months, then its manifest.
|
|
799
|
+
*
|
|
800
|
+
* The manifest is written **last** on purpose. A manifest naming a shard that has not landed yet
|
|
801
|
+
* would send every other device after a 404 on their next run.
|
|
802
|
+
*/
|
|
803
|
+
async push(client) {
|
|
804
|
+
if (!this.store) return { uploaded: 0, unchanged: 0, bytes: 0, shards: [], refused: null }
|
|
805
|
+
// Whose name is this? The shard path is `usage-badge/<device>/<month>`, so a second installation
|
|
806
|
+
// using the same device name does not merge with the first — it **overwrites** its months, and the
|
|
807
|
+
// rows are gone. Nothing about the bytes reveals that, so it has to be caught here, where the answer
|
|
808
|
+
// is "do not write" rather than "write and hope".
|
|
809
|
+
//
|
|
810
|
+
// The owner is recorded in the directory's own manifest. A directory with no recorded owner is one
|
|
811
|
+
// published by a build that did not write the field (or the first publish of anything), and adopting
|
|
812
|
+
// it is what makes an upgrade work; an owner that is a *different* installation is a real clash.
|
|
813
|
+
const remote = (await this.readManifests(client)).find((manifest) => manifest.device === this.deviceId)
|
|
814
|
+
if (remote && remote.instanceId && remote.instanceId !== this.instanceId) {
|
|
815
|
+
this.state.collision = { device: this.deviceId, owner: remote.instanceId, at: this.now() }
|
|
816
|
+
this.log(
|
|
817
|
+
`not publishing: usage-badge/${this.deviceId}/ was published by another installation ` +
|
|
818
|
+
`(${remote.instanceId}, this one is ${this.instanceId}). Pulling is unaffected — everything on the ` +
|
|
819
|
+
`server is merged into this device's history. To publish from this machine, give it a device name ` +
|
|
820
|
+
`that is not already there; "usage-badge/${this.deviceId}/" only decides where new months are written.`,
|
|
821
|
+
)
|
|
822
|
+
return { uploaded: 0, unchanged: 0, bytes: 0, shards: [], refused: 'collision' }
|
|
823
|
+
}
|
|
824
|
+
if (remote && !remote.instanceId && !this.instanceId) {
|
|
825
|
+
// No owner on either side: there is no way to tell the two installations apart, and the whole point
|
|
826
|
+
// of the check is not to guess.
|
|
827
|
+
this.state.collision = { device: this.deviceId, owner: null, at: this.now() }
|
|
828
|
+
this.log(`refusing to publish: usage-badge/${this.deviceId}/ exists and neither side has an identity to compare`)
|
|
829
|
+
return { uploaded: 0, unchanged: 0, bytes: 0, shards: [], refused: 'unidentified' }
|
|
830
|
+
}
|
|
831
|
+
delete this.state.collision
|
|
832
|
+
|
|
833
|
+
const rows = chunkRows(this.store.scan())
|
|
834
|
+
// `state.published` is the manifest's **chunks map**, so it is wrapped back into a manifest shape
|
|
835
|
+
// here. Passing the map itself would leave `previous.chunks` empty, and every run would re-upload
|
|
836
|
+
// every month while looking perfectly healthy.
|
|
837
|
+
const { pending, unchanged, manifest } = planUpload(rows, { chunks: this.state.published ?? {} }, {
|
|
838
|
+
device: this.deviceId,
|
|
839
|
+
label: this.config.deviceLabel ?? '',
|
|
840
|
+
at: this.now(),
|
|
841
|
+
instanceId: this.instanceId,
|
|
842
|
+
})
|
|
843
|
+
|
|
844
|
+
// Is the directory still saying what we last put there? An identity check cannot answer this: a
|
|
845
|
+
// directory copied to another machine carries `sync.json` with it, `instanceId` included, so that
|
|
846
|
+
// machine passes every owner check and then `MOVE`s its months over this one's. The **bytes** answer
|
|
847
|
+
// it — a month we recorded at one hash and the server now holds at another was written by something
|
|
848
|
+
// that was not us, and we have nothing of our own to put there.
|
|
849
|
+
const diverged = Object.entries(this.state.published ?? {}).find(([key, known]) => {
|
|
850
|
+
const remoteEntry = remote?.chunks?.[key]
|
|
851
|
+
if (!remoteEntry || remoteEntry.sha256 === known.sha256) return false
|
|
852
|
+
return !pending.some((item) => item.key === key)
|
|
853
|
+
})
|
|
854
|
+
if (diverged) {
|
|
855
|
+
const [key] = diverged
|
|
856
|
+
this.state.collision = { device: this.deviceId, owner: remote?.instanceId || 'unknown', kind: 'diverged', month: key, at: this.now() }
|
|
857
|
+
this.log(
|
|
858
|
+
`not publishing: usage-badge/${this.deviceId}/${key}.ndjson.zst is not the shard this device last published, ` +
|
|
859
|
+
`and this device has no change of its own for that month — another installation is writing into this directory`,
|
|
860
|
+
)
|
|
861
|
+
return { uploaded: 0, unchanged: 0, bytes: 0, shards: [], refused: 'diverged' }
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
// A month this device published and no longer has any rows for: a session folder was deleted, or a
|
|
865
|
+
// log version that no longer covers that day superseded the rows. The manifest is republished from
|
|
866
|
+
// what is actually held, so **new** devices must stop being told about it — otherwise they import
|
|
867
|
+
// usage its own machine no longer has, which is precisely the "the numbers came back" surprise. The
|
|
868
|
+
// bytes stay on the server; only the claim that this device still holds them is withdrawn.
|
|
869
|
+
const retired = Object.keys(this.state.published ?? {}).filter((key) => !rows.has(key))
|
|
870
|
+
|
|
871
|
+
// The collection comes **before** anything is written into it. `mod_dav_fs` refuses a write whose
|
|
872
|
+
// parent does not exist (403/409, not the 404 an HTTP client might expect), so creating it on the
|
|
873
|
+
// way past would fail the first publish of a device — and mkcol is idempotent anyway, so it costs
|
|
874
|
+
// nothing to do it here.
|
|
875
|
+
await client.mkcol(`${REMOTE_PREFIX}${this.deviceId}/`)
|
|
876
|
+
let bytes = 0
|
|
877
|
+
const shards = []
|
|
878
|
+
for (const item of pending) {
|
|
879
|
+
// Upload under a temporary name and rename: a crash mid-upload leaves a name nobody reads,
|
|
880
|
+
// instead of a shard that looks complete and is half a month.
|
|
881
|
+
const temp = `${item.entry.path}.upload-${process.pid}-${Math.random().toString(16).slice(2, 8)}`
|
|
882
|
+
await client.put(temp, item.body, { contentType: 'application/octet-stream' })
|
|
883
|
+
try {
|
|
884
|
+
await client.move(temp, item.entry.path, { overwrite: true })
|
|
885
|
+
} catch (error) {
|
|
886
|
+
// Leave no debris behind if the rename fails; a leftover temp file is harmless but untidy.
|
|
887
|
+
await client.delete(temp).catch(() => {})
|
|
888
|
+
throw error
|
|
889
|
+
}
|
|
890
|
+
bytes += item.body.length
|
|
891
|
+
shards.push({ key: item.key, bytes: item.body.length, rows: item.rows })
|
|
892
|
+
}
|
|
893
|
+
// The marker exists before the manifest names the directory a device, and the manifest is written
|
|
894
|
+
// **last** on purpose: a manifest naming a shard that has not landed yet would send every other
|
|
895
|
+
// device after a 404 on their next run.
|
|
896
|
+
await client.put(`${REMOTE_PREFIX}${this.deviceId}/${DEVICE_MARKER}`, Buffer.from(`${this.deviceId}\n`))
|
|
897
|
+
let withdrawn = 0
|
|
898
|
+
const withdrawnKeys = []
|
|
899
|
+
for (const key of retired) {
|
|
900
|
+
try {
|
|
901
|
+
// Gone from the manifest *and* from the server: leaving the bytes behind while dropping the claim
|
|
902
|
+
// would be tidier for a manual rescue and worse for everything else — the next reader that finds
|
|
903
|
+
// the manifest is authoritative would see a shard nobody lists. The shard is *this device's own*,
|
|
904
|
+
// which is the one direction where deleting is not the remote deciding anything.
|
|
905
|
+
await client.delete(shardPath(this.deviceId, key))
|
|
906
|
+
delete manifest.chunks[key]
|
|
907
|
+
withdrawn++
|
|
908
|
+
withdrawnKeys.push(key)
|
|
909
|
+
this.log(`withdrew ${key}: this device no longer holds any rows for it`)
|
|
910
|
+
} catch (error) {
|
|
911
|
+
// The month could not be withdrawn, so the manifest is not written at all: half-recording it
|
|
912
|
+
// would leave the claim in place while the shard is gone. Reported, and retried next run.
|
|
913
|
+
return { uploaded: pending.length, unchanged, bytes, shards, manifest: null, withdrawn: 0, refused: null, withdrawError: `could not withdraw ${key}: ${error?.message ?? error}` }
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
await client.put(manifestPath(this.deviceId), Buffer.from(JSON.stringify(manifest, null, 2)), { contentType: 'application/json' })
|
|
917
|
+
return { uploaded: pending.length, unchanged, bytes, shards, manifest, withdrawn, withdrawnKeys }
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/**
|
|
921
|
+
* Reconcile the shared price table.
|
|
922
|
+
*
|
|
923
|
+
* Runs after the shards, because losing a price row is worse than a delayed month: whatever this
|
|
924
|
+
* does, it happens while the shard sync is already recorded as done. The device copy is published
|
|
925
|
+
* first, so that even a conflict the user never looks at leaves both versions on the server.
|
|
926
|
+
*
|
|
927
|
+
* @returns {{action:string, status:string}}
|
|
928
|
+
*/
|
|
929
|
+
async syncPricing(client) {
|
|
930
|
+
const localPath = join(this.dataDir, 'pricing.json')
|
|
931
|
+
// A table this device pulled is still "ours" for the next comparison, so local edits are stamped
|
|
932
|
+
// by the file's own mtime rather than by when the last sync happened — otherwise taking a remote
|
|
933
|
+
// copy would make the *next* run see a local change that nobody made.
|
|
934
|
+
const localStamp = () => {
|
|
935
|
+
try {
|
|
936
|
+
return Math.round(statSync(localPath).mtimeMs) || 0
|
|
937
|
+
} catch {
|
|
938
|
+
return 0
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
let localBytes = null
|
|
942
|
+
try {
|
|
943
|
+
localBytes = readFileSync(localPath)
|
|
944
|
+
} catch {
|
|
945
|
+
// No price table of our own: the built-in template is in force, and there is nothing to publish.
|
|
946
|
+
localBytes = null
|
|
947
|
+
}
|
|
948
|
+
let remoteBytes = null
|
|
949
|
+
try {
|
|
950
|
+
remoteBytes = await client.get(PRICING_FILE)
|
|
951
|
+
} catch (error) {
|
|
952
|
+
this.log(`the shared price table could not be read (${error?.message ?? error}); leaving ours alone`)
|
|
953
|
+
return { action: 'none', status: 'unreadable' }
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
const known = this.state.pricing ?? {}
|
|
957
|
+
const remoteHash = remoteBytes === null ? null : sha256(remoteBytes)
|
|
958
|
+
const plan = planPricing({
|
|
959
|
+
localBytes,
|
|
960
|
+
remoteBytes,
|
|
961
|
+
baseline: known.baseline ?? null,
|
|
962
|
+
localAt: localStamp(),
|
|
963
|
+
remoteAt: known.remoteAt ?? 0,
|
|
964
|
+
device: this.deviceId,
|
|
965
|
+
at: this.now(),
|
|
966
|
+
})
|
|
967
|
+
|
|
968
|
+
if (plan.action === 'none') {
|
|
969
|
+
// `baseline` records what the server had when this device last looked. It is only meaningful
|
|
970
|
+
// when there *is* a remote file: writing the hash of nothing would make the next run treat the
|
|
971
|
+
// baseline as a real version and call a fresh install's first sight of the table a conflict.
|
|
972
|
+
this.state.pricing = remoteHash === null ? { ...known, remoteHash } : { ...known, baseline: remoteHash, remoteHash }
|
|
973
|
+
// In step, but the file on disk may not match — a device that has no table of its own, or whose
|
|
974
|
+
// last run left the server's copy as a conflict file, still needs it written out.
|
|
975
|
+
if (remoteBytes !== null && !sameBytes(localBytes, remoteBytes)) {
|
|
976
|
+
writeFileSync(localPath, remoteBytes)
|
|
977
|
+
return { action: 'pull', status: 'took the server’s copy' }
|
|
978
|
+
}
|
|
979
|
+
return { action: plan.action, status: plan.winner ?? 'in step' }
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
if (plan.conflictFile) {
|
|
983
|
+
// Written locally, not remotely: a conflict copy is for whoever opens this machine. The losing
|
|
984
|
+
// side is always written, and the winner is never applied over a person's edit — the table is
|
|
985
|
+
// hand-written, so a silent overwrite is the one outcome this must not produce.
|
|
986
|
+
const losing = plan.winner === 'remote' ? localBytes ?? Buffer.alloc(0) : remoteBytes ?? Buffer.alloc(0)
|
|
987
|
+
writeFileSync(join(this.dataDir, plan.conflictFile), losing)
|
|
988
|
+
this.log(`the price table changed on both sides; kept the copy that would have been lost as ${plan.conflictFile}`)
|
|
989
|
+
if (plan.winner === 'local') {
|
|
990
|
+
// Ours is the later write, so it is the one that gets published — and the server's copy has
|
|
991
|
+
// just been saved beside it, so nothing was lost either way.
|
|
992
|
+
if (localBytes !== null) {
|
|
993
|
+
await client.put(pricingDevicePath(this.deviceId), localBytes, { contentType: 'application/json' })
|
|
994
|
+
await client.put(PRICING_FILE, localBytes, { contentType: 'application/json' })
|
|
995
|
+
const hash = sha256(localBytes)
|
|
996
|
+
this.state.pricing = { ...known, baseline: hash, remoteHash: hash, remoteAt: this.now() }
|
|
997
|
+
return { action: plan.action, status: 'published ours' }
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
// The server's version is the later write. It is **not** applied over the user's file: their
|
|
1001
|
+
// edit stays where it is, and the server's copy is the conflict file that was just written.
|
|
1002
|
+
this.state.pricing = { ...known, baseline: remoteHash, remoteHash, remoteAt: this.now() }
|
|
1003
|
+
return { action: plan.action, status: 'kept the server’s copy as a conflict file' }
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
if (plan.winner === 'remote') {
|
|
1007
|
+
writeFileSync(localPath, remoteBytes ?? Buffer.alloc(0))
|
|
1008
|
+
this.state.pricing = { ...known, baseline: remoteHash, remoteHash, remoteAt: this.now() }
|
|
1009
|
+
return { action: plan.action, status: 'took the server’s copy' }
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
if (localBytes !== null) {
|
|
1013
|
+
await client.put(pricingDevicePath(this.deviceId), localBytes, { contentType: 'application/json' })
|
|
1014
|
+
await client.put(PRICING_FILE, localBytes, { contentType: 'application/json' })
|
|
1015
|
+
const hash = sha256(localBytes)
|
|
1016
|
+
this.state.pricing = { ...known, baseline: hash, remoteHash: hash, remoteAt: this.now() }
|
|
1017
|
+
return { action: plan.action, status: 'published ours' }
|
|
1018
|
+
}
|
|
1019
|
+
return { action: plan.action, status: 'nothing to publish' }
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
/**
|
|
1023
|
+
* Publish only the price table, taking and giving the lock itself.
|
|
1024
|
+
*
|
|
1025
|
+
* This is what the host half calls a moment after an edit. It is deliberately **not** a full run: a
|
|
1026
|
+
* price edit has nothing to do with shards, and a full run would drag the whole remote listing along
|
|
1027
|
+
* with it. The lock is taken because the edit can land while a full run is in flight, and two writers
|
|
1028
|
+
* on one data directory is the case the lock exists for.
|
|
1029
|
+
*
|
|
1030
|
+
* @param {{lock?:object, instanceId?:string}} [options]
|
|
1031
|
+
* @returns {{result:string, status:string, action?:string, message:string}}
|
|
1032
|
+
*/
|
|
1033
|
+
async pushPricing({ lock = null, instanceId = this.instanceId } = {}) {
|
|
1034
|
+
if (!this.config.enabled || !this.config.url || !this.config.ha1) {
|
|
1035
|
+
return { result: 'skipped', status: 'sync is off or unconfigured', message: 'sync is off or unconfigured' }
|
|
1036
|
+
}
|
|
1037
|
+
const held = lock ?? acquireLock(this.dataDir, { instanceId, log: this.log })
|
|
1038
|
+
if (!held.acquired) return { result: 'locked', status: held.reason, message: held.reason }
|
|
1039
|
+
let client
|
|
1040
|
+
try {
|
|
1041
|
+
client = this.connect()
|
|
1042
|
+
} catch (error) {
|
|
1043
|
+
const message = String(error?.message ?? error)
|
|
1044
|
+
return { result: 'failed', status: message, message }
|
|
1045
|
+
}
|
|
1046
|
+
try {
|
|
1047
|
+
const pricing = await this.syncPricing(client)
|
|
1048
|
+
const message = `prices: ${pricing.status}`
|
|
1049
|
+
// `unreadable` is not a successful reconcile: the shared table could not be read, so whether this
|
|
1050
|
+
// device's edit is safe to publish is unknown — and it is deliberately not published. Recording it
|
|
1051
|
+
// as `ok` would tell the panel everything went fine at the exact moment nothing could be checked.
|
|
1052
|
+
const failed = pricing.status === 'unreadable' || pricing.action === 'failed'
|
|
1053
|
+
this.state.lastSyncAt = this.now()
|
|
1054
|
+
this.state.lastResult = failed ? 'failed' : 'ok'
|
|
1055
|
+
this.state.message = message
|
|
1056
|
+
writeSyncDocument(this.dataDir, { ...this.config, instanceId, state: this.state })
|
|
1057
|
+
return { result: failed ? 'failed' : 'ok', status: pricing.status, action: pricing.action, message }
|
|
1058
|
+
} catch (error) {
|
|
1059
|
+
const message = String(error?.message ?? error)
|
|
1060
|
+
this.log(`pricing push failed: ${message}`)
|
|
1061
|
+
this.state.lastResult = 'failed'
|
|
1062
|
+
this.state.message = `prices: ${message}`
|
|
1063
|
+
writeSyncDocument(this.dataDir, { ...this.config, instanceId, state: this.state })
|
|
1064
|
+
return { result: 'failed', status: message, message }
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
/**
|
|
1069
|
+
* One sync: lock, discover, pull, push, price table, record.
|
|
1070
|
+
*
|
|
1071
|
+
* Every failure is caught and turned into `state.lastResult`, never into a thrown error the caller
|
|
1072
|
+
* has to interpret: the panel shows the sentence, and the local cache is untouched either way.
|
|
1073
|
+
*/
|
|
1074
|
+
async run({ lock = null } = {}) {
|
|
1075
|
+
const started = this.now()
|
|
1076
|
+
if (!this.config.enabled) {
|
|
1077
|
+
return this.finish('skipped', 'sync is turned off', { started })
|
|
1078
|
+
}
|
|
1079
|
+
this.lock = lock ?? acquireLock(this.dataDir, { instanceId: this.instanceId, log: this.log })
|
|
1080
|
+
this.ownsLock = lock === null
|
|
1081
|
+
if (!this.lock.acquired) {
|
|
1082
|
+
// `locked` is not a failure of the sync; it is this process not being the writer. The previous
|
|
1083
|
+
// run's timestamp is deliberately left alone, so the panel does not claim a sync just happened.
|
|
1084
|
+
this.state.lastResult = 'locked'
|
|
1085
|
+
this.state.message = this.lock.reason
|
|
1086
|
+
try {
|
|
1087
|
+
writeSyncDocument(this.dataDir, { ...this.config, instanceId: this.instanceId, state: this.state })
|
|
1088
|
+
} catch (error) {
|
|
1089
|
+
this.log(`sync state could not be saved: ${error?.message ?? error}`)
|
|
1090
|
+
}
|
|
1091
|
+
return { result: 'locked', message: this.lock.reason, ms: this.now() - started }
|
|
1092
|
+
}
|
|
1093
|
+
let client
|
|
1094
|
+
try {
|
|
1095
|
+
client = this.connect()
|
|
1096
|
+
} catch (error) {
|
|
1097
|
+
return this.finish('failed', String(error?.message ?? error), { started })
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
try {
|
|
1101
|
+
const manifests = await this.readManifests(client)
|
|
1102
|
+
// **Pull first, refuse only the push.** An installation whose device name is taken still wants the
|
|
1103
|
+
// history back, and that is the case the feature exists for (reinstall, another machine): the name
|
|
1104
|
+
// is what is in the way, not the data. So the clash is noted here — which is what tells `pull` to
|
|
1105
|
+
// take that directory's shards instead of skipping them as its own — and the refusal itself is
|
|
1106
|
+
// `push`'s, where the decision to write is actually made.
|
|
1107
|
+
const clash = manifests.find((manifest) => manifest.device === this.deviceId && manifest.instanceId && manifest.instanceId !== this.instanceId)
|
|
1108
|
+
if (clash) this.state.collision = { device: this.deviceId, owner: clash.instanceId, at: this.now() }
|
|
1109
|
+
else if (this.state.collision && this.state.collision.device !== this.deviceId) delete this.state.collision
|
|
1110
|
+
// Read the clashing directory only when the clash is a *name* clash. A directory that merely
|
|
1111
|
+
// disagrees with what we recorded is our own, and pulling it back would import a shard we are
|
|
1112
|
+
// refusing to publish — the one case where the two readings of `collision` differ.
|
|
1113
|
+
this.pullOwnDirectory = Boolean(clash)
|
|
1114
|
+
const pulled = await this.pull(client, manifests)
|
|
1115
|
+
this.pullOwnDirectory = false
|
|
1116
|
+
const pushed = await this.push(client)
|
|
1117
|
+
if (pushed.manifest) this.state.published = pushed.manifest.chunks
|
|
1118
|
+
// The price table last: it is the only hand-written thing here, and it should not be touched by
|
|
1119
|
+
// a run that failed to move the usage it was about to move.
|
|
1120
|
+
const pricing = await this.syncPricing(client)
|
|
1121
|
+
const parts = [
|
|
1122
|
+
`pulled ${pulled.fetched} shard(s) / ${pulled.rows} row(s)`,
|
|
1123
|
+
`pushed ${pushed.uploaded} shard(s) / ${Math.round(pushed.bytes / 1024)} KB`,
|
|
1124
|
+
`${pushed.unchanged} month(s) unchanged`,
|
|
1125
|
+
]
|
|
1126
|
+
if (pushed.refused) {
|
|
1127
|
+
// Not `ok`: the months this device folded are not on the server. It is not `failed` either —
|
|
1128
|
+
// the pull worked and nothing was lost. Naming it is what lets the panel say what to do.
|
|
1129
|
+
parts.push(`nothing published: device name "${this.deviceId}" is being written by another installation`)
|
|
1130
|
+
}
|
|
1131
|
+
if (pushed.withdrawError) parts.push(pushed.withdrawError)
|
|
1132
|
+
if (pushed.withdrawn) parts.push(`withdrew ${pushed.withdrawn} month(s) this device no longer holds`)
|
|
1133
|
+
if (pricing.action !== 'none') parts.push(`prices: ${pricing.status}`)
|
|
1134
|
+
if (pulled.failed.length > 0) parts.push(`${pulled.failed.length} shard(s) failed`)
|
|
1135
|
+
const message = parts.join(', ')
|
|
1136
|
+
this.log(`sync: ${message}`)
|
|
1137
|
+
// Four outcomes, each meaning a different thing to the person reading the panel: `ok` moved
|
|
1138
|
+
// everything, `partial` moved almost everything, `clash` deliberately published nothing and needs a
|
|
1139
|
+
// decision from them, `failed` is a problem with the server or the network.
|
|
1140
|
+
const result = pushed.refused ? 'clash' : pulled.failed.length > 0 || pushed.withdrawError ? 'partial' : 'ok'
|
|
1141
|
+
return this.finish(result, message, {
|
|
1142
|
+
started,
|
|
1143
|
+
extra: {
|
|
1144
|
+
pulled: pulled.fetched,
|
|
1145
|
+
rows: pulled.rows,
|
|
1146
|
+
pushed: pushed.uploaded,
|
|
1147
|
+
bytes: pushed.bytes,
|
|
1148
|
+
pricing: pricing.action,
|
|
1149
|
+
failed: pulled.failed,
|
|
1150
|
+
refused: pushed.refused ?? null,
|
|
1151
|
+
withdrawn: pushed.withdrawn ?? 0,
|
|
1152
|
+
},
|
|
1153
|
+
})
|
|
1154
|
+
} catch (error) {
|
|
1155
|
+
const message = String(error?.message ?? error)
|
|
1156
|
+
this.log(`sync failed: ${message}`)
|
|
1157
|
+
return this.finish('failed', message, { started })
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
/**
|
|
1162
|
+
* Record the outcome, persist the state, and return it.
|
|
1163
|
+
*
|
|
1164
|
+
* The state object is **mutated, not replaced**: {@link pull} and {@link push} hold a reference to
|
|
1165
|
+
* it while they work, so assigning a fresh object here would silently drop everything they recorded
|
|
1166
|
+
* — which is exactly how a sync ends up re-importing every shard on every run.
|
|
1167
|
+
*/
|
|
1168
|
+
finish(result, message, { started, extra = {} } = {}) {
|
|
1169
|
+
this.state.lastSyncAt = this.now()
|
|
1170
|
+
this.state.lastResult = result
|
|
1171
|
+
this.state.message = message
|
|
1172
|
+
try {
|
|
1173
|
+
writeSyncDocument(this.dataDir, { ...this.config, instanceId: this.instanceId, state: this.state })
|
|
1174
|
+
} catch (error) {
|
|
1175
|
+
this.log(`sync state could not be saved: ${error?.message ?? error}`)
|
|
1176
|
+
}
|
|
1177
|
+
return { result, message, ms: this.now() - started, ...extra }
|
|
1178
|
+
}
|
|
1179
|
+
}
|