@dinoxx/dinox-cli 1.0.0

package/dist/powersync/tokenIndex.js

@@ -0,0 +1,202 @@
+import crypto from 'node:crypto';
+import { nowIsoUtc } from '../utils/time.js';
+import { stripMarkdown } from '../utils/text.js';
+import { tokenizeWithJieba } from '../utils/tokenize.js';
+const INDEX_STATE_ID = 'c_note';
+const NOTE_LOCAL_FTS_TABLE = 'note_local_fts';
+const DEFAULT_BATCH_SIZE = 300;
+export async function syncNoteTokenIndex(db, options) {
+    const batchSize = Math.max(1, Math.trunc(options?.batchSize ?? DEFAULT_BATCH_SIZE));
+    const nowIso = options?.nowIso ?? nowIsoUtc;
+    const tokenize = options?.tokenize ?? tokenizeWithJieba;
+    await ensureFtsTable(db);
+    const state = await ensureIndexState(db, nowIso);
+    let cursorUpdatedAt = state.cursor_updated_at ?? '';
+    let cursorId = state.cursor_id ?? '';
+    const result = {
+        scanned: 0,
+        reindexed: 0,
+        skipped: 0,
+        removed: 0,
+        ftsRowsUpserted: 0,
+        cursor: {
+            updatedAt: cursorUpdatedAt,
+            id: cursorId,
+        },
+    };
+    while (true) {
+        const rows = await db.getAll(`
+      SELECT
+        id,
+        title,
+        content_md,
+        content_text,
+        tags,
+        created_at,
+        updated_at,
+        is_del,
+        version
+      FROM c_note
+      WHERE
+        COALESCE(updated_at, created_at, '') > ?
+        OR (
+          COALESCE(updated_at, created_at, '') = ?
+          AND id > ?
+        )
+      ORDER BY COALESCE(updated_at, created_at, '') ASC, id ASC
+      LIMIT ?
+      `, [cursorUpdatedAt, cursorUpdatedAt, cursorId, batchSize]);
+        if (rows.length === 0) {
+            break;
+        }
+        for (const row of rows) {
+            result.scanned += 1;
+            const sortKey = noteSortKey(row);
+            const bodySource = selectIndexBodyText(row);
+            const sourceHash = createSourceHash(row, bodySource);
+            const meta = await db.getOptional(`
+        SELECT source_hash
+        FROM note_local_index_meta
+        WHERE id = ?
+        `, [row.id]);
+            if (meta?.source_hash === sourceHash) {
+                result.skipped += 1;
+            }
+            else {
+                const changed = await reindexOneNoteFts(db, row, sourceHash, bodySource, nowIso, tokenize);
+                if (changed.upserted) {
+                    result.reindexed += 1;
+                    result.ftsRowsUpserted += 1;
+                }
+                else {
+                    result.removed += 1;
+                }
+            }
+            cursorUpdatedAt = sortKey;
+            cursorId = row.id;
+        }
+        await db.execute(`
+      UPDATE note_local_index_state
+      SET cursor_updated_at = ?, cursor_id = ?, updated_at = ?
+      WHERE id = ?
+      `, [cursorUpdatedAt, cursorId, nowIso(), INDEX_STATE_ID]);
+    }
+    result.cursor = {
+        updatedAt: cursorUpdatedAt,
+        id: cursorId,
+    };
+    return result;
+}
+async function ensureFtsTable(db) {
+    try {
+        await db.execute(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS ${NOTE_LOCAL_FTS_TABLE}
+      USING fts5(
+        note_id UNINDEXED,
+        title,
+        tags,
+        body,
+        tokenize = 'unicode61'
+      )
+      `, []);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to initialize FTS5 table (${NOTE_LOCAL_FTS_TABLE}): ${message}`);
+    }
+}
+async function ensureIndexState(db, nowIso) {
+    const state = await db.getOptional(`
+    SELECT cursor_updated_at, cursor_id
+    FROM note_local_index_state
+    WHERE id = ?
+    `, [INDEX_STATE_ID]);
+    if (state) {
+        return state;
+    }
+    await db.execute(`
+    INSERT INTO note_local_index_state (id, cursor_updated_at, cursor_id, updated_at)
+    VALUES (?, ?, ?, ?)
+    `, [INDEX_STATE_ID, '', '', nowIso()]);
+    return {
+        cursor_updated_at: '',
+        cursor_id: '',
+    };
+}
+async function reindexOneNoteFts(db, row, sourceHash, bodySource, nowIso, tokenize) {
+    const updatedAt = noteSortKey(row);
+    const indexedAt = nowIso();
+    const deleted = isDeleted(row.is_del);
+    await db.execute(`DELETE FROM ${NOTE_LOCAL_FTS_TABLE} WHERE note_id = ?`, [row.id]);
+    let upserted = false;
+    if (!deleted) {
+        const titleTokens = tokenize(row.title ?? '', 180).join(' ');
+        const tagTokens = tokenize(parseTags(row.tags).join(' '), 180).join(' ');
+        const bodyTokens = tokenize(bodySource, 800).join(' ');
+        await db.execute(`
+      INSERT INTO ${NOTE_LOCAL_FTS_TABLE} (note_id, title, tags, body)
+      VALUES (?, ?, ?, ?)
+      `, [row.id, titleTokens, tagTokens, bodyTokens]);
+        upserted = true;
+    }
+    const existingMeta = await db.getOptional(`
+    SELECT id
+    FROM note_local_index_meta
+    WHERE id = ?
+    `, [row.id]);
+    if (existingMeta) {
+        await db.execute(`
+      UPDATE note_local_index_meta
+      SET source_hash = ?, indexed_at = ?, note_updated_at = ?, is_del = ?
+      WHERE id = ?
+      `, [sourceHash, indexedAt, updatedAt, deleted ? 1 : 0, row.id]);
+    }
+    else {
+        await db.execute(`
+      INSERT INTO note_local_index_meta (id, source_hash, indexed_at, note_updated_at, is_del)
+      VALUES (?, ?, ?, ?, ?)
+      `, [row.id, sourceHash, indexedAt, updatedAt, deleted ? 1 : 0]);
+    }
+    return { upserted };
+}
+function parseTags(raw) {
+    if (!raw) {
+        return [];
+    }
+    try {
+        const parsed = JSON.parse(raw);
+        if (!Array.isArray(parsed)) {
+            return [];
+        }
+        return parsed.map((value) => String(value).trim()).filter(Boolean);
+    }
+    catch {
+        return [];
+    }
+}
+function noteSortKey(row) {
+    return row.updated_at ?? row.created_at ?? '';
+}
+function isDeleted(value) {
+    return typeof value === 'number' && value !== 0;
+}
+function selectIndexBodyText(row) {
+    const contentMd = row.content_md?.trim();
+    if (contentMd) {
+        const markdownText = stripMarkdown(contentMd);
+        if (markdownText) {
+            return markdownText;
+        }
+    }
+    return row.content_text?.trim() ?? '';
+}
+function createSourceHash(row, bodySource) {
+    const normalizedTags = parseTags(row.tags).join('\u0001');
+    const payload = JSON.stringify({
+        title: row.title ?? '',
+        bodySource,
+        tags: normalizedTags,
+        isDel: isDeleted(row.is_del) ? 1 : 0,
+    });
+    return crypto.createHash('sha256').update(payload, 'utf8').digest('hex');
+}

package/dist/powersync/uploader.d.ts

@@ -0,0 +1,7 @@
+import type { UploadHandler } from './connector.js';
+export declare function createPowerSyncUploadHandler(options: {
+    authorization: string;
+    uploadBaseUrl: string;
+    uploadV4Path: string;
+    uploadV2Path: string;
+}): UploadHandler;

package/dist/powersync/uploader.js

@@ -0,0 +1,134 @@
+import { UpdateType } from '@powersync/common';
+const FATAL_RESPONSE_CODES = [/^22...$/, /^23...$/, /^42501$/];
+function shouldFallbackToLegacy(status) {
+    return status === 404 || status === 405;
+}
+function normalizeBaseUrl(baseUrl) {
+    return baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl;
+}
+async function readErrorBody(response) {
+    const contentType = response.headers.get('content-type') ?? '';
+    const text = await response.text().catch(() => '');
+    if (contentType.includes('application/json')) {
+        try {
+            const json = JSON.parse(text);
+            const code = json?.error?.code ?? json?.code;
+            const message = json?.error?.message ?? json?.message;
+            return {
+                code: typeof code === 'string' ? code : undefined,
+                message: typeof message === 'string' ? message : undefined,
+                text,
+            };
+        }
+        catch {
+            return { text };
+        }
+    }
+    return { text };
+}
+function createUploadError(args) {
+    const details = args.message ?? args.text;
+    const message = details
+        ? `PowerSync upload failed (${args.method} ${args.url}) [${args.status}]: ${details}`
+        : `PowerSync upload failed (${args.method} ${args.url}) [${args.status}]`;
+    const error = new Error(message);
+    error.name = 'PowerSyncUploadError';
+    error.code = args.code;
+    error.status = args.status;
+    return error;
+}
+function isFatalUploadError(error) {
+    if (!error || typeof error !== 'object') {
+        return false;
+    }
+    const code = error.code;
+    return typeof code === 'string' && FATAL_RESPONSE_CODES.some(regex => regex.test(code));
+}
+async function dispatchMutation(options) {
+    const response = await fetch(options.url, {
+        method: options.method,
+        headers: {
+            Authorization: options.authorization,
+            'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(options.payload),
+    });
+    if (response.ok) {
+        return;
+    }
+    const body = await readErrorBody(response);
+    throw createUploadError({
+        status: response.status,
+        method: options.method,
+        url: options.url,
+        code: body.code,
+        message: body.message,
+        text: body.text,
+    });
+}
+async function uploadTransaction(args) {
+    const baseUrl = normalizeBaseUrl(args.uploadBaseUrl);
+    const primaryUrl = `${baseUrl}${args.uploadV4Path}`;
+    const fallbackUrl = `${baseUrl}${args.uploadV2Path}`;
+    for (const op of args.transaction.crud) {
+        const record = { table: op.table, data: { ...(op.opData ?? {}), id: op.id } };
+        switch (op.op) {
+            case UpdateType.PUT:
+                try {
+                    await dispatchMutation({ authorization: args.authorization, url: primaryUrl, method: 'PUT', payload: record });
+                }
+                catch (error) {
+                    if (shouldFallbackToLegacy(error?.status)) {
+                        await dispatchMutation({ authorization: args.authorization, url: fallbackUrl, method: 'PUT', payload: record });
+                    }
+                    else {
+                        throw error;
+                    }
+                }
+                break;
+            case UpdateType.PATCH:
+                try {
+                    await dispatchMutation({ authorization: args.authorization, url: primaryUrl, method: 'PATCH', payload: record });
+                }
+                catch (error) {
+                    if (shouldFallbackToLegacy(error?.status)) {
+                        await dispatchMutation({ authorization: args.authorization, url: fallbackUrl, method: 'PATCH', payload: record });
+                    }
+                    else {
+                        throw error;
+                    }
+                }
+                break;
+            case UpdateType.DELETE:
+                await dispatchMutation({ authorization: args.authorization, url: fallbackUrl, method: 'DELETE', payload: record });
+                break;
+            default:
+                break;
+        }
+    }
+}
+export function createPowerSyncUploadHandler(options) {
+    return async (database) => {
+        const transaction = await database.getNextCrudTransaction();
+        if (!transaction) {
+            return;
+        }
+        try {
+            await uploadTransaction({
+                transaction,
+                authorization: options.authorization,
+                uploadBaseUrl: options.uploadBaseUrl,
+                uploadV4Path: options.uploadV4Path,
+                uploadV2Path: options.uploadV2Path,
+            });
+            await transaction.complete();
+        }
+        catch (error) {
+            if (isFatalUploadError(error)) {
+                await transaction.complete();
+                return;
+            }
+            throw error;
+        }
+    };
+}

package/dist/utils/argValue.d.ts

@@ -0,0 +1 @@
+export declare function readStringOrFile(value: string): Promise<string>;

package/dist/utils/argValue.js

@@ -0,0 +1,17 @@
+import fs from 'node:fs/promises';
+import { DinoxError } from './errors.js';
+export async function readStringOrFile(value) {
+    if (!value.startsWith('@')) {
+        return value;
+    }
+    const filePath = value.slice(1);
+    if (!filePath) {
+        throw new DinoxError('Invalid @file syntax: expected @<path>');
+    }
+    try {
+        return await fs.readFile(filePath, 'utf8');
+    }
+    catch (error) {
+        throw new DinoxError(`Failed to read file: ${filePath}`, { cause: error });
+    }
+}

package/dist/utils/errors.d.ts

@@ -0,0 +1,10 @@
+export declare class DinoxError extends Error {
+    readonly exitCode: number;
+    readonly details?: unknown;
+    constructor(message: string, options?: {
+        exitCode?: number;
+        cause?: unknown;
+        details?: unknown;
+    });
+}
+export declare function isDinoxError(error: unknown): error is DinoxError;

package/dist/utils/errors.js

@@ -0,0 +1,17 @@
+export class DinoxError extends Error {
+    exitCode;
+    details;
+    constructor(message, options) {
+        super(message);
+        this.name = 'DinoxError';
+        this.exitCode = options?.exitCode ?? 1;
+        this.details = options?.details;
+        if (options?.cause !== undefined) {
+            // Node 16+ supports Error.cause.
+            this.cause = options.cause;
+        }
+    }
+}
+export function isDinoxError(error) {
+    return Boolean(error) && typeof error === 'object' && error.name === 'DinoxError';
+}

package/dist/utils/id.d.ts

@@ -0,0 +1 @@
+export declare function generateId(): string;

package/dist/utils/id.js

@@ -0,0 +1,4 @@
+import { uuidv7 } from 'uuidv7';
+export function generateId() {
+    return uuidv7();
+}

package/dist/utils/output.d.ts

@@ -0,0 +1,2 @@
+export declare function toYaml(value: unknown): string;
+export declare function printYaml(value: unknown): void;

package/dist/utils/output.js

@@ -0,0 +1,10 @@
+import { stringify as stringifyYaml } from 'yaml';
+export function toYaml(value) {
+    return stringifyYaml(value, {
+        lineWidth: 0,
+        indent: 2,
+    });
+}
+export function printYaml(value) {
+    process.stdout.write(toYaml(value));
+}

package/dist/utils/redact.d.ts

@@ -0,0 +1 @@
+export declare function redactAuthorization(authorization?: string): string | undefined;

package/dist/utils/redact.js

@@ -0,0 +1,10 @@
+export function redactAuthorization(authorization) {
+    if (!authorization) {
+        return undefined;
+    }
+    const trimmed = authorization.trim();
+    if (trimmed.length <= 12) {
+        return '***';
+    }
+    return `${trimmed.slice(0, 6)}...${trimmed.slice(-4)}`;
+}

package/dist/utils/text.d.ts

@@ -0,0 +1 @@
+export declare function stripMarkdown(markdown: string): string;

package/dist/utils/text.js

@@ -0,0 +1,35 @@
+import remarkGfm from 'remark-gfm';
+import remarkParse from 'remark-parse';
+import remarkStringify from 'remark-stringify';
+import stripMarkdownPlugin from 'strip-markdown';
+import { unified } from 'unified';
+const markdownProcessor = unified()
+    .use(remarkParse)
+    .use(remarkGfm)
+    .use(stripMarkdownPlugin)
+    .use(remarkStringify);
+export function stripMarkdown(markdown) {
+    if (!markdown.trim()) {
+        return '';
+    }
+    try {
+        const plain = String(markdownProcessor.processSync(markdown).value);
+        return normalizeWhitespace(decodeHtmlEntities(plain));
+    }
+    catch {
+        return normalizeWhitespace(markdown);
+    }
+}
+function decodeHtmlEntities(value) {
+    return value
+        .replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
+        .replace(/&#([0-9]+);/g, (_, dec) => String.fromCodePoint(Number.parseInt(dec, 10)))
+        .replace(/&/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&apos;/g, "'");
+}
+function normalizeWhitespace(value) {
+    return value.replace(/\s+/g, ' ').trim();
+}

package/dist/utils/time.d.ts

@@ -0,0 +1 @@
+export declare function nowIsoUtc(): string;

package/dist/utils/time.js

@@ -0,0 +1,3 @@
+export function nowIsoUtc() {
+    return new Date().toISOString();
+}

package/dist/utils/tiptapMarkdown.d.ts

@@ -0,0 +1,6 @@
+import { type JSONContent } from '@tiptap/core';
+export type MarkdownConversion = {
+    contentJson: JSONContent;
+    contentText: string;
+};
+export declare function convertMarkdownToNoteContent(markdown: string): MarkdownConversion;

package/dist/utils/tiptapMarkdown.js

@@ -0,0 +1,149 @@
+import { Editor } from '@tiptap/core';
+import { Image } from '@tiptap/extension-image';
+import { ListItem } from '@tiptap/extension-list';
+import { Markdown } from '@tiptap/markdown';
+import { StarterKit } from '@tiptap/starter-kit';
+import { TableKit } from '@tiptap/extension-table';
+import { UniqueID, generateUniqueIds } from '@tiptap/extension-unique-id';
+import { DinoxError } from './errors.js';
+import { generateId } from './id.js';
+const EMPTY_PARAGRAPH = { type: 'paragraph', content: [] };
+const CONTENT_PREVIEW_MAX_LENGTH = 400;
+const markdownExtension = Markdown.configure({
+    markedOptions: {
+        gfm: true,
+    },
+});
+const uniqueIdExtension = UniqueID.configure({
+    attributeName: 'id',
+    types: [
+        'heading',
+        'paragraph',
+        'blockquote',
+        'codeBlock',
+        'listItem',
+        'table',
+        'tableRow',
+        'tableCell',
+        'tableHeader',
+    ],
+    generateID: () => generateId(),
+    updateDocument: false,
+});
+function ensureLeadingParagraph(content) {
+    if (content.length === 0) {
+        return [EMPTY_PARAGRAPH];
+    }
+    if (content[0]?.type === 'paragraph') {
+        return content;
+    }
+    return [EMPTY_PARAGRAPH, ...content];
+}
+// Tiptap markdown parser may produce listItem that starts with blockquote/list.
+// ProseMirror schema requires paragraph as first child (paragraph block*).
+const SafeListItem = ListItem.extend({
+    parseMarkdown: (token, helpers) => {
+        if (token.type !== 'list_item') {
+            return [];
+        }
+        let content = [];
+        if (token.tokens && token.tokens.length > 0) {
+            const hasParagraphTokens = token.tokens.some((inner) => inner.type === 'paragraph');
+            if (hasParagraphTokens) {
+                content = helpers.parseChildren(token.tokens);
+            }
+            else {
+                const firstToken = token.tokens[0];
+                if (firstToken?.type === 'text' && firstToken.tokens && firstToken.tokens.length > 0) {
+                    content = [
+                        {
+                            type: 'paragraph',
+                            content: helpers.parseInline(firstToken.tokens),
+                        },
+                    ];
+                    if (token.tokens.length > 1) {
+                        content.push(...helpers.parseChildren(token.tokens.slice(1)));
+                    }
+                }
+                else {
+                    content = helpers.parseChildren(token.tokens);
+                }
+            }
+        }
+        return {
+            type: 'listItem',
+            content: ensureLeadingParagraph(content),
+        };
+    },
+});
+const tiptapExtensions = [
+    StarterKit.configure({ listItem: false }),
+    SafeListItem,
+    Image,
+    TableKit,
+    markdownExtension,
+    uniqueIdExtension,
+];
+function sanitizeNode(node) {
+    if (!node || typeof node !== 'object') {
+        return node;
+    }
+    const next = { ...node };
+    if (Array.isArray(node.content)) {
+        next.content = node.content
+            .filter((child) => !!child && typeof child === 'object')
+            .map((child) => sanitizeNode(child));
+    }
+    if (next.type === 'listItem') {
+        const content = Array.isArray(next.content) ? next.content : [];
+        if (content.length === 0) {
+            next.content = [EMPTY_PARAGRAPH];
+        }
+        else if (content[0]?.type !== 'paragraph') {
+            next.content = [EMPTY_PARAGRAPH, ...content];
+        }
+    }
+    return next;
+}
+function normalizeContentText(text) {
+    const normalized = (text ?? '').replace(/\r\n/g, '\n');
+    const lines = normalized
+        .split('\n')
+        .map((line) => line.trim())
+        .filter(Boolean)
+        .filter((line) => !/^\[[^\]]+\]:\s*(?:<)?https?:\/\/\S+/i.test(line))
+        .filter((line) => !/^(https?:\/\/|www\.)\S+$/i.test(line));
+    const cleaned = lines
+        .join(' ')
+        .replace(/\[\^[^\]]+\]/g, '')
+        .replace(/\[(\d{1,4})\]/g, '')
+        .replace(/\bhttps?:\/\/\S+/gi, '')
+        .replace(/\bwww\.\S+/gi, '')
+        .replace(/\s{2,}/g, ' ')
+        .trim();
+    if (cleaned.length <= CONTENT_PREVIEW_MAX_LENGTH) {
+        return cleaned;
+    }
+    return cleaned.slice(0, CONTENT_PREVIEW_MAX_LENGTH);
+}
+export function convertMarkdownToNoteContent(markdown) {
+    let editor = null;
+    try {
+        editor = new Editor({
+            editable: false,
+            extensions: tiptapExtensions,
+            content: markdown,
+            contentType: 'markdown',
+        });
+        const sanitized = sanitizeNode(editor.getJSON());
+        const contentJson = generateUniqueIds(sanitized, tiptapExtensions);
+        const contentText = normalizeContentText(editor.getText());
+        return { contentJson, contentText };
+    }
+    catch (error) {
+        throw new DinoxError('Failed to convert markdown to TipTap JSON', { cause: error });
+    }
+    finally {
+        editor?.destroy();
+    }
+}

package/dist/utils/tokenize.d.ts

@@ -0,0 +1 @@
+export declare function tokenizeWithJieba(text: string, limit?: number): string[];