markdown-notes-engine 1.0.2 → 2.0.1

package/lib/backend/db/connection.js

@@ -0,0 +1,127 @@
+/**
+ * Database Connection Manager
+ * Handles PostgreSQL connection pooling using the postgres package
+ */
+
+import postgres from 'postgres';
+import { readFileSync } from 'fs';
+
+export class DatabaseConnection {
+  constructor(config) {
+    // Support both connection string and individual parameters
+    let connectionString;
+
+    if (config.connectionString) {
+      // Use provided connection string
+      connectionString = config.connectionString;
+    } else {
+      // Build connection string from individual parameters
+      const user = config.user;
+      const password = config.password;
+      const host = config.host || 'localhost';
+      const port = config.port || 5432;
+      const database = config.database;
+
+      connectionString = `postgres://${user}:${password}@${host}:${port}/${database}`;
+    }
+
+    // postgres package options
+    const options = {
+      max: config.maxConnections || 20,
+      idle_timeout: config.idleTimeout || 30,
+      connect_timeout: config.connectionTimeout || 10,
+      fetch_types: false,
+    };
+
+    // SSL configuration
+    if (config.ssl !== undefined) {
+      if (typeof config.ssl === 'boolean') {
+        options.ssl = config.ssl ? 'prefer' : false;
+      } else {
+        options.ssl = config.ssl;
+      }
+    } else if (config.connectionString || (config.host && config.host !== 'localhost' && !config.host.startsWith('127.'))) {
+      // Auto-enable SSL for remote connections (use 'prefer' for better compatibility)
+      options.ssl = 'prefer';
+    }
+
+    // Create postgres connection
+    this.sql = postgres(connectionString, options);
+  }
+
+  /**
+   * Execute a query using tagged template
+   * This wrapper provides compatibility with the old API
+   * @param {string} text - SQL query text
+   * @param {Array} [params] - Query parameters
+   * @returns {Promise<Object>} Query result with rows property
+   */
+  async query(text, params = []) {
+    // Convert parameterized query to postgres format
+    // Replace $1, $2, etc. with actual values
+    const result = await this.sql.unsafe(text, params);
+
+    // Return in pg-compatible format
+    return { rows: result };
+  }
+
+  /**
+   * Get a client for transactions
+   * @returns {Promise<Object>} Transaction client
+   */
+  async getClient() {
+    // Reserve a connection from the pool for transaction safety
+    const reserved = await this.sql.reserve();
+
+    return {
+      query: async (text, params = []) => {
+        const result = await reserved.unsafe(text, params);
+        return { rows: result };
+      },
+      release: () => {
+        // Release the reserved connection back to the pool
+        reserved.release();
+      }
+    };
+  }
+
+  /**
+   * Initialize database schema
+   * @param {string} schemaPath - Path to schema SQL file
+   * @returns {Promise<void>}
+   */
+  async initializeSchema(schemaPath) {
+    const schema = readFileSync(schemaPath, 'utf-8');
+    await this.query(schema);
+
+    // Initialize default branch if it doesn't exist
+    const result = await this.query(
+      'SELECT name FROM branches WHERE name = $1',
+      ['main']
+    );
+
+    if (result.rows.length === 0) {
+      // Create initial commit
+      const commitResult = await this.query(
+        `INSERT INTO commits (message, author)
+         VALUES ($1, $2)
+         RETURNING id`,
+        ['Initial commit', 'system']
+      );
+
+      // Create main branch pointing to initial commit
+      await this.query(
+        'INSERT INTO branches (name, commit_id) VALUES ($1, $2)',
+        ['main', commitResult.rows[0].id]
+      );
+    }
+  }
+
+  /**
+   * Close all connections
+   * @returns {Promise<void>}
+   */
+  async close() {
+    await this.sql.end();
+  }
+}

package/lib/backend/db/schema.sql

@@ -0,0 +1,144 @@
+-- Git-like Version Control Schema for PostgreSQL
+
+-- Content-addressable blob storage
+-- Each unique file content is stored once, identified by its SHA-256 hash
+CREATE TABLE IF NOT EXISTS blobs (
+  hash TEXT PRIMARY KEY,
+  content TEXT NOT NULL,
+  size INTEGER NOT NULL,
+  created_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_blobs_created_at ON blobs(created_at);
+
+-- Commits represent snapshots in time
+-- Similar to Git commits, they have a parent and point to a tree
+CREATE TABLE IF NOT EXISTS commits (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  parent_id UUID REFERENCES commits(id),
+  message TEXT NOT NULL,
+  author TEXT NOT NULL,
+  created_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_commits_parent_id ON commits(parent_id);
+CREATE INDEX IF NOT EXISTS idx_commits_created_at ON commits(created_at);
+
+-- Trees map file paths to blob hashes for each commit
+-- This allows us to reconstruct the file system state at any commit
+CREATE TABLE IF NOT EXISTS trees (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  commit_id UUID NOT NULL REFERENCES commits(id) ON DELETE CASCADE,
+  path TEXT NOT NULL,
+  blob_hash TEXT NOT NULL REFERENCES blobs(hash),
+  created_at TIMESTAMP DEFAULT NOW(),
+  UNIQUE(commit_id, path)
+);
+
+CREATE INDEX IF NOT EXISTS idx_trees_commit_id ON trees(commit_id);
+CREATE INDEX IF NOT EXISTS idx_trees_path ON trees(path);
+CREATE INDEX IF NOT EXISTS idx_trees_blob_hash ON trees(blob_hash);
+
+-- Branches are named pointers to commits
+-- Similar to Git branches, they move as new commits are made
+CREATE TABLE IF NOT EXISTS branches (
+  name TEXT PRIMARY KEY,
+  commit_id UUID NOT NULL REFERENCES commits(id),
+  created_at TIMESTAMP DEFAULT NOW(),
+  updated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_branches_commit_id ON branches(commit_id);
+
+-- Media references (for tracking S3/R2 uploaded files)
+-- Links media files to markdown content for cleanup and tracking
+CREATE TABLE IF NOT EXISTS media (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  url TEXT NOT NULL UNIQUE,
+  type TEXT NOT NULL CHECK (type IN ('image', 'video')),
+  s3_key TEXT NOT NULL,
+  blob_hash TEXT REFERENCES blobs(hash),
+  created_at TIMESTAMP DEFAULT NOW(),
+  updated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_media_blob_hash ON media(blob_hash);
+CREATE INDEX IF NOT EXISTS idx_media_type ON media(type);
+
+-- Full-text search index for markdown content
+-- Enables fast search across all note content
+CREATE TABLE IF NOT EXISTS search_index (
+  blob_hash TEXT PRIMARY KEY REFERENCES blobs(hash) ON DELETE CASCADE,
+  content_tsvector TSVECTOR NOT NULL,
+  created_at TIMESTAMP DEFAULT NOW(),
+  updated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_search_content ON search_index USING GIN(content_tsvector);
+
+-- Migration: Add timestamp columns to existing tables
+-- These statements handle adding columns to tables that already exist
+
+-- Add created_at to trees if it doesn't exist
+DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM information_schema.columns
+    WHERE table_name = 'trees' AND column_name = 'created_at'
+  ) THEN
+    ALTER TABLE trees ADD COLUMN created_at TIMESTAMP DEFAULT NOW();
+    UPDATE trees SET created_at = NOW() WHERE created_at IS NULL;
+    CREATE INDEX IF NOT EXISTS idx_trees_created_at ON trees(created_at);
+  END IF;
+END $$;
+
+-- Add created_at to branches if it doesn't exist
+DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM information_schema.columns
+    WHERE table_name = 'branches' AND column_name = 'created_at'
+  ) THEN
+    ALTER TABLE branches ADD COLUMN created_at TIMESTAMP DEFAULT NOW();
+    UPDATE branches SET created_at = NOW() WHERE created_at IS NULL;
+  END IF;
+END $$;
+
+-- Create index on branches.updated_at (this column should exist in new or old schemas)
+CREATE INDEX IF NOT EXISTS idx_branches_updated_at ON branches(updated_at);
+
+-- Add updated_at to media if it doesn't exist
+DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM information_schema.columns
+    WHERE table_name = 'media' AND column_name = 'updated_at'
+  ) THEN
+    ALTER TABLE media ADD COLUMN updated_at TIMESTAMP DEFAULT NOW();
+    UPDATE media SET updated_at = created_at WHERE updated_at IS NULL;
+  END IF;
+END $$;
+
+-- Create index on media.created_at (this column should exist in new or old schemas)
+CREATE INDEX IF NOT EXISTS idx_media_created_at ON media(created_at);
+
+-- Add created_at and updated_at to search_index if they don't exist
+DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM information_schema.columns
+    WHERE table_name = 'search_index' AND column_name = 'created_at'
+  ) THEN
+    ALTER TABLE search_index ADD COLUMN created_at TIMESTAMP DEFAULT NOW();
+    UPDATE search_index SET created_at = NOW() WHERE created_at IS NULL;
+  END IF;
+
+  IF NOT EXISTS (
+    SELECT 1 FROM information_schema.columns
+    WHERE table_name = 'search_index' AND column_name = 'updated_at'
+  ) THEN
+    ALTER TABLE search_index ADD COLUMN updated_at TIMESTAMP DEFAULT NOW();
+    UPDATE search_index SET updated_at = NOW() WHERE updated_at IS NULL;
+    CREATE INDEX IF NOT EXISTS idx_search_updated_at ON search_index(updated_at);
+  END IF;
+END $$;

package/lib/backend/github.js

@@ -3,9 +3,9 @@
  * Handles all GitHub API operations for note management
  */
 
-const { Octokit } = require('@octokit/rest');
+import { Octokit } from '@octokit/rest';
 
-class GitHubClient {
+export class GitHubClient {
   constructor({ token, owner, repo, branch = 'main' }) {
     this.octokit = new Octokit({ auth: token });
     this.owner = owner;
@@ -314,5 +314,3 @@ class GitHubClient {
     return root;
   }
 }
-
-module.exports = { GitHubClient };

package/lib/backend/index.js

@@ -4,22 +4,36 @@
  * Creates an Express router with all note-taking API endpoints
  */
 
-const express = require('express');
-const { GitHubClient } = require('./github');
-const { StorageClient } = require('./storage');
-const { MarkdownRenderer } = require('./markdown');
-const notesRoutes = require('./routes/notes');
-const uploadRoutes = require('./routes/upload');
-const searchRoutes = require('./routes/search');
+import express from 'express';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { StorageClient } from './storage.js';
+import { MarkdownRenderer } from './markdown.js';
+import { DatabaseConnection } from './db/connection.js';
+import { VersionControlClient } from './version-control.js';
+import notesRoutes from './routes/notes.js';
+import uploadRoutes from './routes/upload.js';
+import searchRoutes from './routes/search.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
 
 /**
  * Creates a configured notes router
  * @param {Object} config - Configuration object
- * @param {Object} config.github - GitHub configuration
- * @param {string} config.github.token - GitHub personal access token
- * @param {string} config.github.owner - Repository owner
- * @param {string} config.github.repo - Repository name
- * @param {string} [config.github.branch='main'] - Repository branch (defaults to 'main')
+ * @param {Object} [config.database] - Database configuration (recommended)
+ * @param {string} [config.database.host] - PostgreSQL host
+ * @param {number} [config.database.port] - PostgreSQL port
+ * @param {string} config.database.database - Database name
+ * @param {string} config.database.user - Database user
+ * @param {string} config.database.password - Database password
+ * @param {string} [config.database.branch='main'] - Version control branch
+ * @param {string} [config.database.author='user'] - Default commit author
+ * @param {Object} [config.github] - GitHub configuration (legacy)
+ * @param {string} [config.github.token] - GitHub personal access token
+ * @param {string} [config.github.owner] - Repository owner
+ * @param {string} [config.github.repo] - Repository name
+ * @param {string} [config.github.branch='main'] - Repository branch
  * @param {Object} config.storage - Storage configuration (R2 or S3)
  * @param {string} config.storage.type - 'r2' or 's3'
  * @param {string} config.storage.accountId - Account ID (R2) or region (S3)
@@ -29,26 +43,55 @@ const searchRoutes = require('./routes/search');
  * @param {string} config.storage.publicUrl - Public URL for accessing files
  * @param {Object} [config.options] - Optional configuration
  * @param {boolean} [config.options.autoUpdateReadme=true] - Auto-update README on note save
- * @returns {express.Router} Configured Express router
+ * @param {boolean} [config.options.autoInitSchema=true] - Auto-initialize database schema
+ * @returns {Promise<express.Router>} Configured Express router
  */
-function createNotesRouter(config) {
-  if (!config.github || !config.github.token || !config.github.owner || !config.github.repo) {
-    throw new Error('GitHub configuration is required: token, owner, and repo');
-  }
-
+export async function createNotesRouter(config) {
   if (!config.storage) {
     throw new Error('Storage configuration is required');
   }
 
   const router = express.Router();
 
-  // Initialize clients
-  const githubClient = new GitHubClient({
-    token: config.github.token,
-    owner: config.github.owner,
-    repo: config.github.repo,
-    branch: config.github.branch || 'main'
-  });
+  // Add JSON body parser middleware to the router
+  router.use(express.json());
+
+  let versionControlClient;
+
+  // Choose between database-backed version control or GitHub
+  if (config.database) {
+    // Use PostgreSQL-backed version control
+    const db = new DatabaseConnection(config.database);
+
+    // Initialize schema if needed
+    const options = config.options || {};
+    if (options.autoInitSchema !== false) {
+      const schemaPath = path.join(__dirname, 'db', 'schema.sql');
+      await db.initializeSchema(schemaPath);
+    }
+
+    versionControlClient = new VersionControlClient(
+      db,
+      config.database.branch || 'main',
+      config.database.author || 'user'
+    );
+  } else if (config.github) {
+    // Legacy: Use GitHub for version control
+    if (!config.github.token || !config.github.owner || !config.github.repo) {
+      throw new Error('GitHub configuration requires: token, owner, and repo');
+    }
+
+    // Lazy load GitHubClient to avoid requiring @octokit/rest when using database mode
+    const { GitHubClient } = await import('./github.js');
+    versionControlClient = new GitHubClient({
+      token: config.github.token,
+      owner: config.github.owner,
+      repo: config.github.repo,
+      branch: config.github.branch || 'main'
+    });
+  } else {
+    throw new Error('Either database or github configuration is required');
+  }
 
   const storageClient = new StorageClient(config.storage);
   const markdownRenderer = new MarkdownRenderer();
@@ -57,7 +100,7 @@ function createNotesRouter(config) {
   // Middleware to attach clients to request
   router.use((req, res, next) => {
     req.notesEngine = {
-      githubClient,
+      githubClient: versionControlClient, // Keep name for backward compatibility
       storageClient,
       markdownRenderer,
       options
@@ -72,5 +115,3 @@ function createNotesRouter(config) {
 
   return router;
 }
-
-module.exports = { createNotesRouter };

package/lib/backend/markdown.js

@@ -3,11 +3,11 @@
  * Renders markdown to HTML with syntax highlighting
  */
 
-const { marked } = require('marked');
-const { markedHighlight } = require('marked-highlight');
-const hljs = require('highlight.js');
+import { marked } from 'marked';
+import { markedHighlight } from 'marked-highlight';
+import hljs from 'highlight.js';
 
-class MarkdownRenderer {
+export class MarkdownRenderer {
   constructor() {
     // Configure marked with syntax highlighting
     marked.use(markedHighlight({
@@ -58,5 +58,3 @@ class MarkdownRenderer {
     return marked(markdown);
   }
 }
-
-module.exports = { MarkdownRenderer };

package/lib/backend/routes/notes.js

@@ -3,15 +3,18 @@
  * Handles note CRUD operations
  */
 
-const express = require('express');
+import express from 'express';
 const router = express.Router();
 
+// JSON body parser middleware
+router.use(express.json());
+
 // Get file structure
 router.get('/structure', async (req, res) => {
   try {
     const { githubClient } = req.notesEngine;
     const structure = await githubClient.getFileStructure();
-    res.json({ structure });
+    res.json(structure);
   } catch (error) {
     console.error('Error fetching structure:', error);
     res.status(500).json({ error: 'Failed to fetch file structure' });
@@ -40,6 +43,33 @@ router.get('/note', async (req, res) => {
   }
 });
 
+// Get file history (commits)
+router.get('/history', async (req, res) => {
+  try {
+    const { path } = req.query;
+    if (!path) {
+      return res.status(400).json({ error: 'Path is required' });
+    }
+
+    const { githubClient } = req.notesEngine;
+
+    // Check if the client supports getFileHistory (database mode)
+    if (typeof githubClient.getFileHistory === 'function') {
+      const history = await githubClient.getFileHistory(path);
+      res.json(history);
+    } else {
+      // GitHub mode doesn't support file history yet
+      res.status(501).json({
+        error: 'File history not supported with GitHub backend',
+        message: 'Use database backend for file history support'
+      });
+    }
+  } catch (error) {
+    console.error('Error fetching file history:', error);
+    res.status(500).json({ error: 'Failed to fetch file history' });
+  }
+});
+
 // Save note
 router.post('/note', async (req, res) => {
   try {
@@ -83,7 +113,8 @@ router.post('/note', async (req, res) => {
 // Delete note
 router.delete('/note', async (req, res) => {
   try {
-    const { path } = req.body;
+    // Support both query params and body for flexibility
+    const path = req.query.path || req.body?.path;
 
     if (!path) {
       return res.status(400).json({ error: 'Path is required' });
@@ -194,4 +225,4 @@ async function updateReadme(githubClient) {
   await githubClient.saveFile('README.md', readmeContent, sha);
 }
 
-module.exports = router;
+export default router;

package/lib/backend/routes/search.js

@@ -3,7 +3,7 @@
  * Handles note searching
  */
 
-const express = require('express');
+import express from 'express';
 const router = express.Router();
 
 // Search notes
@@ -25,4 +25,4 @@ router.get('/search', async (req, res) => {
   }
 });
 
-module.exports = router;
+export default router;

package/lib/backend/routes/upload.js

@@ -3,8 +3,8 @@
  * Handles image and video uploads
  */
 
-const express = require('express');
-const fileUpload = require('express-fileupload');
+import express from 'express';
+import fileUpload from 'express-fileupload';
 const router = express.Router();
 
 // File upload middleware
@@ -27,7 +27,16 @@ router.post('/upload-image', async (req, res) => {
       folder
     );
 
-    res.json({ imageUrl });
+    // Extract path and filename from URL
+    const path = imageUrl.replace(storageClient.publicUrl + '/', '');
+    const filename = path.split('/').pop();
+
+    res.json({
+      success: true,
+      url: imageUrl,
+      path,
+      filename
+    });
   } catch (error) {
     console.error('Error uploading image:', error);
     res.status(500).json({ error: 'Failed to upload image' });
@@ -50,7 +59,16 @@ router.post('/upload-image-base64', async (req, res) => {
     const { storageClient } = req.notesEngine;
     const imageUrl = await storageClient.uploadImage(buffer, filename, folder);
 
-    res.json({ imageUrl });
+    // Extract path and filename from URL
+    const path = imageUrl.replace(storageClient.publicUrl + '/', '');
+    const uploadedFilename = path.split('/').pop();
+
+    res.json({
+      success: true,
+      url: imageUrl,
+      path,
+      filename: uploadedFilename
+    });
   } catch (error) {
     console.error('Error uploading image:', error);
     res.status(500).json({ error: 'Failed to upload image' });
@@ -74,7 +92,16 @@ router.post('/upload-video', async (req, res) => {
       folder
     );
 
-    res.json({ videoUrl });
+    // Extract path and filename from URL
+    const path = videoUrl.replace(storageClient.publicUrl + '/', '');
+    const filename = path.split('/').pop();
+
+    res.json({
+      success: true,
+      url: videoUrl,
+      path,
+      filename
+    });
   } catch (error) {
     console.error('Error uploading video:', error);
     res.status(500).json({ error: 'Failed to upload video' });
@@ -119,4 +146,4 @@ router.delete('/video', async (req, res) => {
   }
 });
 
-module.exports = router;
+export default router;

package/lib/backend/storage.js

@@ -3,9 +3,9 @@
  * Handles file uploads to R2 or S3
  */
 
-const { S3Client, PutObjectCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3');
+import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
 
-class StorageClient {
+export class StorageClient {
   constructor(config) {
     this.config = config;
     this.publicUrl = config.publicUrl;
@@ -117,5 +117,3 @@ class StorageClient {
     return contentTypes[ext] || 'application/octet-stream';
   }
 }
-
-module.exports = { StorageClient };