mikser-io-drive 0.9.0

package/lib/staged-writes.js

@@ -0,0 +1,95 @@
+import { rename, unlink, open } from 'node:fs/promises'
+import { randomBytes } from 'node:crypto'
+import path from 'node:path'
+
+// Make a PUT atomic from the filesystem's point of view.
+//
+// @nephele/adapter-file-system writes straight to the destination:
+//
+//     const handle = await fsp.open(this.absolutePath, 'w')   // truncates NOW
+//     input.pipe(handle.createWriteStream())
+//
+// Measured against a trickling 512KB upload, that has two consequences, and
+// the second is data loss:
+//
+//   1. The file is visible while it grows — 65536, 131072, … 524288 bytes.
+//      These folders are mikser sources, so the watcher can import a
+//      half-written file and render a truncated page.
+//   2. An interrupted overwrite DESTROYS what was there. `'w'` truncates at
+//      open, so a 1600-byte file whose upload died after 196608 bytes was
+//      left as 196608 bytes of the new content. Not the old file, not an
+//      error — a corrupted file, silently.
+//
+// Both go away by writing to a sibling temp file and renaming on success.
+// rename(2) within a directory is atomic: the watcher sees the file appear
+// complete or not at all, and a failed upload leaves the original untouched
+// because it was never opened for writing.
+//
+// A sibling rather than the OS temp dir, because rename is only atomic within
+// one filesystem and /tmp is frequently another mount.
+export function stageWrites(resource, { onFailure } = {}) {
+    if (typeof resource?.setStream !== 'function') return resource
+
+    return new Proxy(resource, {
+        get(target, prop, receiver) {
+            if (prop !== 'setStream') {
+                const value = Reflect.get(target, prop, receiver)
+                return typeof value === 'function' ? value.bind(target) : value
+            }
+
+            return async function setStream(input, user, ...rest) {
+                const finalPath = target.absolutePath
+                // No path to stage next to — a virtual or non-filesystem
+                // resource. Fall through rather than guess.
+                if (!finalPath) return target.setStream(input, user, ...rest)
+
+                const tempPath = path.join(
+                    path.dirname(finalPath),
+                    `.${path.basename(finalPath)}.${randomBytes(6).toString('hex')}.part`,
+                )
+
+                let handle
+                try {
+                    handle = await open(tempPath, 'wx')
+                    await new Promise((resolve, reject) => {
+                        const stream = handle.createWriteStream()
+                        stream.on('error', reject)
+                        stream.on('close', resolve)
+                        input.on('error', (err) => { stream.destroy(); reject(err) })
+                        input.pipe(stream)
+                    })
+                    await handle.close()
+                    handle = null
+
+                    // The atomic moment. Everything before this is invisible
+                    // to the watcher because the temp name is dot-prefixed
+                    // and suffixed .part; everything after is a complete file.
+                    await rename(tempPath, finalPath)
+
+                    // The adapter caches these off the old file.
+                    target.etag = undefined
+                    target.stats = undefined
+                } catch (err) {
+                    await handle?.close().catch(() => {})
+                    await unlink(tempPath).catch(() => {})
+                    onFailure?.(err, finalPath)
+                    throw err
+                }
+            }
+        },
+    })
+}
+
+// Wrap an adapter so every resource it hands out stages its writes.
+export function withStagedWrites(adapter, options = {}) {
+    const wrap = (method) => async (...args) => stageWrites(await adapter[method](...args), options)
+    return new Proxy(adapter, {
+        get(target, prop, receiver) {
+            if (prop === 'getResource' || prop === 'newResource' || prop === 'newCollection') {
+                return wrap(prop)
+            }
+            const value = Reflect.get(target, prop, receiver)
+            return typeof value === 'function' ? value.bind(target) : value
+        },
+    })
+}

package/litmus-server.mjs

@@ -0,0 +1,74 @@
+// Target for an external WebDAV compliance run.
+//
+//     node litmus-server.mjs                  # boots on :8399
+//     TESTS=props litmus http://127.0.0.1:8399/webdav/compliant/ litmus litmus-pw
+//
+// litmus (the neon project's suite) is not a dependency — install it from your
+// distro. Scores are recorded in the README; this file is what produced them,
+// kept so they can be reproduced rather than taken on faith.
+//
+// Boots the WebDAV server on a fixed port for an external compliance run.
+// Two endpoints, because they should NOT score the same:
+//
+//   /webdav/default    properties+locks 'emulate' — the shipped default
+//   /webdav/compliant  properties+locks 'meta-files' — full storage
+//
+// litmus's `props` suite sets a dead property and reads it back. 'emulate'
+// reports success without storing, so it is expected to fail those — that is
+// the documented trade, and a compliance suite is exactly the thing that
+// should say so out loud.
+import { mkdtemp, writeFile, mkdir } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import bcrypt from 'bcryptjs'
+import { runtime } from 'mikser-io'
+import { auth } from 'mikser-io-auth'
+import { webdav } from './index.js'
+
+const PORT = Number(process.env.PORT ?? 8399)
+
+const dir = await mkdtemp(path.join(tmpdir(), 'litmus-'))
+// A folder PER endpoint. Sharing one made the two suites contaminate each
+// other's leftover /litmus/ collection, which showed up as a suite aborting
+// at `begin` — a harness bug that looked like a server bug.
+await mkdir(path.join(dir, 'default'), { recursive: true })
+await mkdir(path.join(dir, 'compliant'), { recursive: true })
+await mkdir(path.join(dir, 'runtime'), { recursive: true })
+await writeFile(path.join(dir, 'users.htpasswd'), `litmus:${bcrypt.hashSync('litmus-pw', 10)}\n`)
+
+const { default: express } = await import('express')
+const app = express()
+runtime.options = { ...runtime.options, app, workingFolder: dir, runtimeFolder: path.join(dir, 'runtime') }
+runtime.config = {}
+runtime.engine = { logger: { info(){}, warn(){}, error(){}, debug(){}, trace(){}, fatal(){} } }
+
+const identity = auth({})   // no capability map → authenticated users unscoped
+const plugin = webdav({
+    endpoints: {
+        default:   { folder: 'default' },
+        compliant: { folder: 'compliant', properties: 'meta-files', locks: 'meta-files' },
+    },
+    auth: identity,
+})
+
+const load = [], loaded = []
+const core = {
+    runtime,
+    onLoad: (c) => load.push(c),
+    onLoaded: (c) => loaded.push(c),
+    useLogger: () => ({ info(){}, warn(){}, error(){}, debug(){}, trace(){} }),
+}
+identity(core)
+plugin(core)
+for (const c of load) await c()
+for (const c of loaded) await c()
+
+const server = app.listen(PORT, () => {
+    console.log(`litmus target ready on http://127.0.0.1:${PORT}`)
+    console.log(`  working folder: ${dir}`)
+    console.log(`  credentials:    litmus / litmus-pw`)
+    console.log(`  endpoints:      /webdav/default  /webdav/compliant`)
+})
+// litmus uploads and holds connections; do not let node's 5-minute cap
+// truncate a suite mid-run.
+server.requestTimeout = 30 * 60 * 1000

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "mikser-io-drive",
+  "version": "0.9.0",
+  "description": "WebDAV for mikser-io. Exposes working-folder directories as WebDAV endpoints via Nephele, authenticated through mikser-io-auth's htpasswd identity — so content can be edited from Finder, Explorer or any DAV client and the build picks the change up.",
+  "main": "index.js",
+  "type": "module",
+  "scripts": {
+    "test": "node --no-warnings --test --test-reporter=spec 'test/**/*.test.js'"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/almero-digital-marketing/mikser-io-drive.git"
+  },
+  "keywords": [
+    "mikser",
+    "mikser-io",
+    "webdav",
+    "dav",
+    "nephele"
+  ],
+  "author": "",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/almero-digital-marketing/mikser-io-drive/issues"
+  },
+  "homepage": "https://github.com/almero-digital-marketing/mikser-io-drive#readme",
+  "peerDependencies": {
+    "mikser-io": "^9.7.0",
+    "zod": "^4.0.0"
+  },
+  "dependencies": {
+    "@nephele/adapter-file-system": "1.0.0-alpha.67",
+    "@nephele/plugin-read-only": "1.0.0-alpha.67",
+    "nephele": "1.0.0-alpha.67"
+  },
+  "devDependencies": {
+    "bcryptjs": "^3.0.0",
+    "express": "^5.2.1",
+    "globby": "^15.0.0",
+    "mikser-io": "file:../mikser-io",
+    "mikser-io-auth": "file:../mikser-io-auth",
+    "webdav": "^5.10.0",
+    "zod": "^4.0.0"
+  }
+}

package/test/atomic-writes.test.js

@@ -0,0 +1,140 @@
+import { describe, it, before, after } from 'node:test'
+import assert from 'node:assert/strict'
+import { mkdtemp, writeFile, rm, mkdir, stat, readFile, readdir } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import bcrypt from 'bcryptjs'
+
+import { runtime } from 'mikser-io'
+import { auth } from 'mikser-io-auth'
+import { drive } from '../index.js'
+
+// These two properties are the reason lib/staged-writes.js exists. The
+// underlying adapter writes with open(path, 'w') straight to the destination,
+// which was measured to (1) expose a growing partial file and (2) destroy the
+// previous contents when an upload is interrupted. Both are silent, and both
+// matter because these folders are mikser sources.
+let server, port, dir, content
+const AUTH = 'Basic ' + Buffer.from('alice:alice-pw').toString('base64')
+const CHUNK = 65536
+const CHUNKS = 8
+
+before(async () => {
+    dir = await mkdtemp(path.join(tmpdir(), 'mikser-drive-atomic-'))
+    content = path.join(dir, 'documents')
+    await mkdir(content, { recursive: true })
+    await mkdir(path.join(dir, 'runtime'), { recursive: true })
+    await writeFile(path.join(dir, 'users.htpasswd'), `alice:${bcrypt.hashSync('alice-pw', 10)}\n`)
+
+    const { default: express } = await import('express')
+    const app = express()
+    runtime.options = { ...runtime.options, app, workingFolder: dir, runtimeFolder: path.join(dir, 'runtime') }
+    runtime.config = { ...runtime.config }
+    runtime.engine = { ...runtime.engine, logger: { info(){}, warn(){}, error(){}, debug(){}, trace(){}, fatal(){} } }
+
+    const identity = auth({})
+    const plugin = drive({ endpoints: { content: { folder: 'documents' } }, auth: identity })
+    const load = [], loaded = []
+    const core = {
+        runtime,
+        onLoad: (c) => load.push(c),
+        onLoaded: (c) => loaded.push(c),
+        useLogger: () => ({ info(){}, warn(){}, error(){}, debug(){}, trace(){} }),
+    }
+    identity(core)
+    plugin(core)
+    for (const c of load) await c()
+    for (const c of loaded) await c()
+
+    server = await new Promise(r => { const s = app.listen(0, () => r(s)) })
+    port = server.address().port
+})
+
+after(async () => {
+    await new Promise(r => server?.close(r))
+    await rm(dir, { recursive: true, force: true })
+})
+
+// A body that trickles, so the upload is observable while in flight.
+function slowBody({ chunks = CHUNKS, size = CHUNK, gapMs = 120, abortAt = null } = {}) {
+    let i = 0
+    return new ReadableStream({
+        async pull(controller) {
+            if (abortAt != null && i === abortAt) {
+                controller.error(new Error('client vanished'))
+                return
+            }
+            if (i >= chunks) { controller.close(); return }
+            controller.enqueue(new Uint8Array(size).fill(65 + (i % 26)))
+            i++
+            await new Promise(r => setTimeout(r, gapMs))
+        },
+    })
+}
+
+const put = (name, body) => fetch(`http://127.0.0.1:${port}/drive/content/${name}`, {
+    method: 'PUT', headers: { authorization: AUTH }, body, duplex: 'half',
+})
+const sizeOf = async (p) => { try { return (await stat(p)).size } catch { return null } }
+
+describe('a partial upload is never visible at the destination', () => {
+    it('the target path does not exist until the upload completes', async () => {
+        const target = path.join(content, 'big.md')
+        const seen = []
+        const poll = setInterval(async () => seen.push(await sizeOf(target)), 100)
+        const res = await put('big.md', slowBody())
+        clearInterval(poll)
+        await new Promise(r => setTimeout(r, 50))
+
+        assert.ok([201, 204].includes(res.status), `unexpected ${res.status}`)
+        assert.equal(await sizeOf(target), CHUNK * CHUNKS, 'the whole file landed')
+
+        // Without staging, this sequence was [65536, 131072, … 524288].
+        const partial = seen.filter(s => s !== null && s > 0 && s < CHUNK * CHUNKS)
+        assert.deepEqual(partial, [], `a growing file was visible: ${JSON.stringify(seen)}`)
+        assert.ok(seen.length >= 3, 'the poll must have run while the upload was in flight')
+    })
+
+    it('leaves no staging file behind on success', async () => {
+        await put('clean.md', slowBody({ chunks: 2, gapMs: 20 }))
+        const leftovers = (await readdir(content)).filter(f => f.endsWith('.part'))
+        assert.deepEqual(leftovers, [])
+    })
+})
+
+describe('an interrupted overwrite leaves the original intact', () => {
+    it('does not truncate or corrupt the file that was there', async () => {
+        // The measured failure without staging: a 1600-byte file became
+        // 196608 bytes of the new content. Not the old file, not an error.
+        const target = path.join(content, 'precious.md')
+        const original = '# original content that matters\n'.repeat(50)
+        await writeFile(target, original)
+
+        await assert.rejects(() => put('precious.md', slowBody({ abortAt: 3 })))
+        await new Promise(r => setTimeout(r, 200))
+
+        assert.equal(await readFile(target, 'utf8'), original)
+    })
+
+    it('cleans up the staging file after a failed upload', async () => {
+        await assert.rejects(() => put('doomed.md', slowBody({ abortAt: 2 })))
+        await new Promise(r => setTimeout(r, 200))
+        const leftovers = (await readdir(content)).filter(f => f.endsWith('.part'))
+        assert.deepEqual(leftovers, [], 'a failed upload must not leave a .part file')
+        assert.equal(await sizeOf(path.join(content, 'doomed.md')), null,
+                     'and must not create the destination at all')
+    })
+})
+
+describe('atomicWrites: false restores the adapter behaviour', () => {
+    it('is available as an escape hatch', async () => {
+        // Documented rather than recommended: the point is that the default
+        // is the safe one and the unsafe one has to be asked for.
+        const { withStagedWrites, stageWrites } = await import('../index.js')
+        assert.equal(typeof withStagedWrites, 'function')
+        assert.equal(typeof stageWrites, 'function')
+        // A resource with no setStream passes through untouched.
+        const plain = { absolutePath: '/x' }
+        assert.equal(stageWrites(plain), plain)
+    })
+})

package/test/derived.test.js

@@ -0,0 +1,164 @@
+import { describe, it, before, after } from 'node:test'
+import assert from 'node:assert/strict'
+import { mkdtemp, rm, mkdir, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { spawn } from 'node:child_process'
+import path from 'node:path'
+import { createRequire } from 'node:module'
+
+// `derived` is a passthrough of what the assets plugin stamped, and it was the
+// one field in the add response I could not verify by unit test: it only exists
+// when a real preset matched and a real derivative was produced. So this runs
+// the whole pipeline — files → assets → renderPreset → sharp — against a live
+// server and asks the tool.
+//
+// The read is three lines. What this pins is that those three lines name the
+// field the pipeline actually writes, which is the half that silently rots when
+// the plugin changes shape.
+//
+// Two mistakes made while building it, both of which produce SILENCE rather
+// than an error and are worth knowing:
+//
+//   - a preset is a MODULE with a default export that does the work, not an
+//     inline options object. Without the default, renderPreset resolves
+//     `runtime.preset` to undefined and every render throws.
+//   - `match` runs against the entity ID. `outputFolder` prefixes `name` and
+//     `meta.url` but NOT `id`, so a glob written against the URL shape
+//     (/media/**) matches nothing, produces no variant, and reports only that
+//     none matched.
+
+// sharp is mikser-io's dependency, not this package's — the fixture needs it
+// because a preset does the actual image work. Resolved from there rather than
+// from here, and the fixture's node_modules is built from mikser-io's so the
+// spawned build finds it too.
+const require_ = createRequire(import.meta.url)
+const ENGINE = path.resolve(import.meta.dirname, '..', '..', 'mikser-io')
+const hasSharp = (() => {
+    try { require_.resolve('sharp', { paths: [ENGINE] }); return true } catch { return false }
+})()
+
+const PORT = 4211
+const PNG = Buffer.from(
+    'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==',
+    'base64')
+
+let dir, server
+
+async function rpc(token, method, params, session) {
+    const headers = {
+        'content-type': 'application/json',
+        accept: 'application/json, text/event-stream',
+        ...(token ? { authorization: `Bearer ${token}` } : {}),
+        ...(session.id ? { 'mcp-session-id': session.id } : {}),
+    }
+    const res = await fetch(`http://127.0.0.1:${PORT}/mcp`,
+        { method: 'POST', headers, body: JSON.stringify({ jsonrpc: '2.0', id: session.n++, method, params }) })
+    if (res.headers.get('mcp-session-id')) session.id = res.headers.get('mcp-session-id')
+    const text = await res.text()
+    const frames = text.split('\n').filter(l => l.startsWith('data:')).map(l => JSON.parse(l.slice(5)))
+    return (frames.at(-1) ?? JSON.parse(text)).result
+}
+
+before(async (t) => {
+    if (!hasSharp) return
+    dir = await mkdtemp(path.join(tmpdir(), 'mikser-derived-'))
+    await mkdir(path.join(dir, 'media'), { recursive: true })
+    await mkdir(path.join(dir, 'presets'), { recursive: true })
+    await mkdir(path.join(dir, 'documents'), { recursive: true })
+
+    // node_modules by symlink, so the fixture resolves the working copies.
+    const here = path.resolve(import.meta.dirname, '..')
+    await mkdir(path.join(dir, 'node_modules'), { recursive: true })
+    const { symlink, readdir } = await import('node:fs/promises')
+    // Both donors: this package's own deps, plus the engine's — which is where
+    // sharp lives, and a preset cannot run without it.
+    for (const donor of [path.join(here, 'node_modules'), path.join(ENGINE, 'node_modules')]) {
+        for (const entry of await readdir(donor).catch(() => [])) {
+            await symlink(path.join(donor, entry), path.join(dir, 'node_modules', entry)).catch(() => {})
+        }
+    }
+    for (const pkg of ['mikser-io', 'mikser-io-mcp', 'mikser-io-drive']) {
+        await rm(path.join(dir, 'node_modules', pkg), { recursive: true, force: true })
+        await symlink(path.resolve(here, '..', pkg), path.join(dir, 'node_modules', pkg)).catch(() => {})
+    }
+
+    // A preset is a module whose DEFAULT export does the work.
+    await writeFile(path.join(dir, 'presets/web.js'),
+        "import sharp from 'sharp'\n"
+        + 'export const revision = 1\n'
+        + "export const format = 'webp'\n"
+        + 'export const options = { width: 8 }\n'
+        + 'export default ({ entity: { source, destination }, options }) =>\n'
+        + '    sharp(source).resize(options).webp().toFile(destination)\n')
+
+    // `match` against the ID shape, which the files plugin builds from the path
+    // inside filesFolder — NOT the URL shape.
+    await writeFile(path.join(dir, 'mikser.config.js'),
+        "import { files, assets, renderPreset } from 'mikser-io'\n"
+        + "import { mcp } from 'mikser-io-mcp'\n"
+        + "import { drive } from 'mikser-io-drive'\n"
+        + 'export default async () => ({\n'
+        + '    plugins: [\n'
+        + "        mcp({ base: '' }),\n"
+        + "        files({ filesFolder: 'media', outputFolder: 'media' }),\n"
+        + "        assets({ assetsFolder: 'derived', presetsFolder: 'presets',\n"
+        + "                 presets: { web: { match: ['/files/**/*.png'] } } }),\n"
+        + '        renderPreset(),\n'
+        + "        drive({ allowRemote: true, endpoints: { media: { folder: 'media' } } }),\n"
+        + '    ],\n'
+        + '})\n')
+
+    server = spawn(process.execPath, [
+        path.join(dir, 'node_modules/mikser-io/app.js'),
+        '--working-folder', dir, '--output-folder', 'out', '--runtime-folder', 'runtime',
+        '--watch', '--server', String(PORT), '--url', `http://127.0.0.1:${PORT}`,
+    ], { cwd: dir, stdio: 'ignore' })
+
+    // Wait for it to answer rather than sleeping a guess.
+    for (let i = 0; i < 60; i++) {
+        try {
+            const res = await fetch(`http://127.0.0.1:${PORT}/mcp`, { method: 'POST' })
+            if (res.status) break
+        } catch { await new Promise(r => setTimeout(r, 1000)) }
+    }
+    await new Promise(r => setTimeout(r, 3000))
+})
+
+after(async () => {
+    server?.kill()
+    if (dir) await rm(dir, { recursive: true, force: true })
+})
+
+describe('derived — the preset passthrough', { skip: hasSharp ? false : 'sharp is not installed' }, () => {
+    it('reports the variant a preset actually produced', async () => {
+        const session = { id: null, n: 1 }
+        await rpc(null, 'initialize',
+            { protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 't', version: '0' } }, session)
+        await fetch(`http://127.0.0.1:${PORT}/mcp`, {
+            method: 'POST',
+            headers: { 'content-type': 'application/json', accept: 'application/json, text/event-stream',
+                       'mcp-session-id': session.id },
+            body: JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }),
+        })
+
+        const result = await rpc(null, 'tools/call', {
+            name: 'mikser_drive_add',
+            arguments: { endpoint: 'media', folder: 'shots',
+                         files: [{ name: 'hero.png', base64: PNG.toString('base64'), mime: 'image/png' }] },
+        }, session)
+        assert.equal(result?.isError, undefined, result?.content?.[0]?.text)
+
+        const body = JSON.parse(result.content[0].text)
+        const stored = body.files[0]
+        assert.equal(stored.imported, true, 'the cycle must have imported it')
+        assert.equal(stored.reference, '/media/shots/hero.png')
+
+        // The field this whole fixture exists for.
+        assert.ok(stored.derived, `expected derived variants, got: ${stored.derivedNote ?? '(nothing)'}`)
+        assert.ok(stored.derived.web, `expected a "web" variant, got ${JSON.stringify(stored.derived)}`)
+        assert.match(stored.derived.web, /\.webp$/,
+            'the preset declares format webp, so the variant URL should carry that extension')
+        assert.notEqual(stored.derived.web, stored.reference,
+            'the variant must be a different URL from the original, or nothing was derived')
+    })
+})