@moxn/kb-migrate 0.4.33 → 0.4.35

package/dist/index.js

@@ -950,15 +950,9 @@ async function importNotionDatabase(client, dbImport, log, options, preCreatedDb
 }
 /**
  * Slugify a string for use as a tag path segment.
- * Similar to page slug but for the tag hierarchy.
+ * Delegates to the shared notion slugify so tags and page paths normalize identically.
  */
 function slugifyTagPath(s) {
-    return (s
-        .toLowerCase()
-        .trim()
-        .replace(/[^\w\s-]/g, '')
-        .replace(/[\s_]+/g, '-')
-        .replace(/-+/g, '-')
-        .replace(/^-|-$/g, '') || 'untitled');
+    return slugify(s);
 }
 program.parse();

package/dist/sources/local.d.ts

@@ -6,6 +6,21 @@
 import { MigrationSource, type SourceConfig } from './base.js';
 import type { ExtractedDocument } from '../types.js';
 import type { DateFilter } from '../date-filter.js';
+/**
+ * Sanitize a single path segment for ltree compatibility.
+ * Server's validatePathForLtree only allows [a-zA-Z0-9-]. We:
+ *   - normalize NFKD + strip combining marks so "résumé" → "resume"
+ *   - replace anything not in [a-zA-Z0-9-] with "-"
+ *   - collapse runs of "-" and trim leading/trailing "-"
+ * Falls back to "untitled" when the result is empty (e.g. "___.md").
+ */
+export declare function sanitizePathSegment(segment: string): string;
+/**
+ * Convert a relative file path (e.g. "subdir/My_Doc.v2.md") into a
+ * server-acceptable KB path (e.g. "subdir/My-Doc-v2"). Strips the file
+ * extension and sanitizes each path segment.
+ */
+export declare function filePathToKbPath(relativePath: string): string;
 export interface LocalSourceConfig extends SourceConfig {
     /** Directory path to scan for documents */
     directory: string;

package/dist/sources/local.js

@@ -10,6 +10,33 @@ import { glob } from 'glob';
 import { unified } from 'unified';
 import remarkParse from 'remark-parse';
 import { MigrationSource } from './base.js';
+/**
+ * Sanitize a single path segment for ltree compatibility.
+ * Server's validatePathForLtree only allows [a-zA-Z0-9-]. We:
+ *   - normalize NFKD + strip combining marks so "résumé" → "resume"
+ *   - replace anything not in [a-zA-Z0-9-] with "-"
+ *   - collapse runs of "-" and trim leading/trailing "-"
+ * Falls back to "untitled" when the result is empty (e.g. "___.md").
+ */
+export function sanitizePathSegment(segment) {
+    const cleaned = segment
+        .normalize('NFKD')
+        .replace(/[̀-ͯ]/g, '')
+        .replace(/[^a-zA-Z0-9-]/g, '-')
+        .replace(/-+/g, '-')
+        .replace(/^-+|-+$/g, '');
+    return cleaned.length > 0 ? cleaned : 'untitled';
+}
+/**
+ * Convert a relative file path (e.g. "subdir/My_Doc.v2.md") into a
+ * server-acceptable KB path (e.g. "subdir/My-Doc-v2"). Strips the file
+ * extension and sanitizes each path segment.
+ */
+export function filePathToKbPath(relativePath) {
+    const parsed = path.parse(relativePath);
+    const dirParts = parsed.dir ? parsed.dir.split(path.sep) : [];
+    return [...dirParts, parsed.name].map(sanitizePathSegment).join('/');
+}
 /**
  * Local filesystem migration source
  */
@@ -94,9 +121,7 @@ export class LocalSource extends MigrationSource {
         // Detect KB path collisions (e.g., doc.md and doc.mdx both map to /doc)
         const pathToFiles = new Map();
         for (const file of uniqueFiles) {
-            const parsed = path.parse(file);
-            const dirParts = parsed.dir ? parsed.dir.split(path.sep) : [];
-            const kbPath = [...dirParts, parsed.name].join('/').replace(/ /g, '-');
+            const kbPath = filePathToKbPath(file);
             const existing = pathToFiles.get(kbPath) || [];
             existing.push(file);
             pathToFiles.set(kbPath, existing);
@@ -120,10 +145,11 @@ export class LocalSource extends MigrationSource {
         const parsed = path.parse(relativePath);
         const dirParts = parsed.dir ? parsed.dir.split(path.sep) : [];
         const rawPath = [...dirParts, parsed.name].join('/');
-        // Sanitize: replace spaces with hyphens (server rejects paths with spaces)
-        const docPath = rawPath.replace(/ /g, '-');
+        // Sanitize each segment: server's ltree validation only allows [a-zA-Z0-9-].
+        // Spaces, underscores, dots, and other special chars are replaced with hyphens.
+        const docPath = filePathToKbPath(relativePath);
         if (docPath !== rawPath) {
-            console.warn(`  ⚠ Path sanitized: "${rawPath}" → "${docPath}" (spaces replaced with hyphens)`);
+            console.warn(`  ⚠ Path sanitized: "${rawPath}" → "${docPath}" (unsupported chars replaced with hyphens)`);
         }
         // Derive name from filename
         const name = parsed.name

package/dist/sources/local.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/sources/local.test.js

@@ -0,0 +1,63 @@
+import { describe, it, expect } from 'vitest';
+import { sanitizePathSegment, filePathToKbPath } from './local.js';
+describe('sanitizePathSegment', () => {
+    it('replaces spaces with hyphens', () => {
+        expect(sanitizePathSegment('My Doc')).toBe('My-Doc');
+    });
+    it('replaces underscores with hyphens (ltree forbids _)', () => {
+        expect(sanitizePathSegment('my_doc')).toBe('my-doc');
+    });
+    it('replaces periods with hyphens (ltree forbids .)', () => {
+        expect(sanitizePathSegment('my.doc.v2')).toBe('my-doc-v2');
+    });
+    it('handles mixed underscores, periods, and spaces', () => {
+        expect(sanitizePathSegment('My_Doc.v2 final')).toBe('My-Doc-v2-final');
+    });
+    it('collapses runs of separators', () => {
+        expect(sanitizePathSegment('foo___bar...baz')).toBe('foo-bar-baz');
+    });
+    it('replaces other special characters with hyphens', () => {
+        expect(sanitizePathSegment('foo@bar!')).toBe('foo-bar');
+    });
+    it('handles parentheses, ampersands, apostrophes', () => {
+        expect(sanitizePathSegment("Q3 Plan (final)")).toBe('Q3-Plan-final');
+        expect(sanitizePathSegment("Tom's & Jerry's")).toBe('Tom-s-Jerry-s');
+    });
+    it('strips accents via NFKD normalization', () => {
+        expect(sanitizePathSegment('résumé')).toBe('resume');
+        expect(sanitizePathSegment('café')).toBe('cafe');
+    });
+    it('replaces unsupported scripts with hyphens', () => {
+        expect(sanitizePathSegment('файл')).toBe('untitled');
+    });
+    it('trims leading/trailing hyphens', () => {
+        expect(sanitizePathSegment('_foo_')).toBe('foo');
+    });
+    it('preserves case', () => {
+        expect(sanitizePathSegment('MyDoc')).toBe('MyDoc');
+    });
+    it('falls back to "untitled" for empty result', () => {
+        expect(sanitizePathSegment('___')).toBe('untitled');
+        expect(sanitizePathSegment('!!!')).toBe('untitled');
+    });
+});
+describe('filePathToKbPath', () => {
+    it('strips file extension', () => {
+        expect(filePathToKbPath('readme.md')).toBe('readme');
+    });
+    it('sanitizes underscores in filename', () => {
+        expect(filePathToKbPath('my_notes.md')).toBe('my-notes');
+    });
+    it('sanitizes periods in filename', () => {
+        expect(filePathToKbPath('release.v2.md')).toBe('release-v2');
+    });
+    it('sanitizes each directory segment', () => {
+        expect(filePathToKbPath('eng_docs/my.spec/readme.md')).toBe('eng-docs/my-spec/readme');
+    });
+    it('handles paths with both underscores and periods together', () => {
+        expect(filePathToKbPath('release_notes/v1.2.md')).toBe('release-notes/v1-2');
+    });
+    it('handles parens and special chars in filename', () => {
+        expect(filePathToKbPath('Q3 Plan (final).md')).toBe('Q3-Plan-final');
+    });
+});

package/dist/sources/notion.d.ts

@@ -127,6 +127,12 @@ export interface PageTreeResult {
 export declare function buildPageTree(pages: NotionPage[], options?: BuildPageTreeOptions): PageTreeResult;
 /**
  * Slugify a page title for use as a KB path segment.
- * Lowercase, spaces→hyphens, strip special chars, ltree-compatible.
+ *
+ * Server's validatePathForLtree only allows [a-zA-Z0-9-]. We:
+ *   - lowercase
+ *   - normalize NFKD + strip combining marks so "résumé" → "resume"
+ *   - replace anything not in [a-z0-9-] with "-"
+ *   - collapse runs of "-" and trim leading/trailing "-"
+ * Falls back to "untitled" when the result is empty (e.g. emoji-only or non-Latin titles).
  */
 export declare function slugify(title: string): string;

package/dist/sources/notion.js

@@ -596,15 +596,21 @@ function getParentPageId(page) {
 // ============================================
 /**
  * Slugify a page title for use as a KB path segment.
- * Lowercase, spaces→hyphens, strip special chars, ltree-compatible.
+ *
+ * Server's validatePathForLtree only allows [a-zA-Z0-9-]. We:
+ *   - lowercase
+ *   - normalize NFKD + strip combining marks so "résumé" → "resume"
+ *   - replace anything not in [a-z0-9-] with "-"
+ *   - collapse runs of "-" and trim leading/trailing "-"
+ * Falls back to "untitled" when the result is empty (e.g. emoji-only or non-Latin titles).
  */
 export function slugify(title) {
-    return (title
+    const cleaned = title
+        .normalize('NFKD')
+        .replace(/[̀-ͯ]/g, '')
         .toLowerCase()
-        .trim()
-        .replace(/[^\w\s-]/g, '') // Remove special chars (keep word chars, spaces, hyphens)
-        .replace(/[\s_]+/g, '-') // Spaces and underscores → hyphens (ltree forbids _ and .)
-        .replace(/-+/g, '-') // Collapse multiple hyphens
-        .replace(/^-|-$/g, '') || // Trim leading/trailing hyphens
-        'untitled');
+        .replace(/[^a-z0-9-]/g, '-')
+        .replace(/-+/g, '-')
+        .replace(/^-+|-+$/g, '');
+    return cleaned.length > 0 ? cleaned : 'untitled';
 }

package/dist/sources/notion.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/sources/notion.test.js

@@ -0,0 +1,45 @@
+import { describe, it, expect } from 'vitest';
+import { slugify } from './notion.js';
+describe('slugify (notion)', () => {
+    it('lowercases', () => {
+        expect(slugify('MyDoc')).toBe('mydoc');
+    });
+    it('replaces spaces with hyphens', () => {
+        expect(slugify('My Doc')).toBe('my-doc');
+    });
+    it('replaces underscores with hyphens (ltree forbids _)', () => {
+        expect(slugify('my_doc')).toBe('my-doc');
+    });
+    it('replaces periods with hyphens (ltree forbids .)', () => {
+        expect(slugify('release.v2')).toBe('release-v2');
+    });
+    it('handles parens, ampersands, apostrophes', () => {
+        expect(slugify('Q3 Plan (final)')).toBe('q3-plan-final');
+        expect(slugify("Tom's & Jerry's")).toBe('tom-s-jerry-s');
+    });
+    it('strips accents via NFKD normalization', () => {
+        expect(slugify('résumé')).toBe('resume');
+        expect(slugify('café')).toBe('cafe');
+        expect(slugify('naïve')).toBe('naive');
+    });
+    it('replaces non-Latin scripts with hyphens', () => {
+        expect(slugify('файл')).toBe('untitled');
+        expect(slugify('日本語')).toBe('untitled');
+    });
+    it('preserves digits', () => {
+        expect(slugify('Q3-2024')).toBe('q3-2024');
+    });
+    it('collapses runs of separators', () => {
+        expect(slugify('foo___bar...baz')).toBe('foo-bar-baz');
+    });
+    it('trims leading/trailing hyphens', () => {
+        expect(slugify('-foo-')).toBe('foo');
+        expect(slugify('  hello  ')).toBe('hello');
+    });
+    it('falls back to "untitled" for empty result', () => {
+        expect(slugify('')).toBe('untitled');
+        expect(slugify('___')).toBe('untitled');
+        expect(slugify('!!!')).toBe('untitled');
+        expect(slugify('🎉')).toBe('untitled');
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moxn/kb-migrate",
-  "version": "0.4.33",
+  "version": "0.4.35",
   "description": "Migration tool for importing documents into Moxn Knowledge Base from local files, Notion, Google Docs, and more",
   "type": "module",
   "main": "dist/index.js",