@realtimex/realtimex-alchemy 1.0.44 → 1.0.46

package/dist/CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.46] - 2026-01-26
+
+### Fixed
+- **Browser Mining**: Fixed Safari history extraction query and timestamp conversion (CFAbsoluteTime).
+- **Browser Mining**: Updated Chrome/Edge timestamp logic to use BigInt for high-precision microsecond conversion, resolving potential checkpoint drift.
+- **Sync Logic**: Switched history extraction order to Ascending (oldest first) to ensure checkpoints correctly advance from the start date.
+
+## [1.0.45] - 2026-01-26
+
+### Security
+- **Migration Tool**: Enforced Access Token authentication for the `POST /api/migrate` endpoint and `migrate.sh` script. Database passwords are no longer accepted as a security hardening measure.
+
 ## [1.0.44] - 2026-01-26
 
 ### Added

package/dist/api/index.js

@@ -30,10 +30,13 @@ app.get('/health', (req, res) => {
 });
 // Run database migrations (SSE stream)
 app.post('/api/migrate', (req, res) => {
-    const { projectId, dbPassword, accessToken } = req.body;
+    const { projectId, accessToken } = req.body;
     if (!projectId) {
         return res.status(400).json({ error: 'Project ID is required' });
     }
+    if (!accessToken) {
+        return res.status(400).json({ error: 'Access token is required' });
+    }
     // Set up SSE for streaming output
     res.setHeader('Content-Type', 'text/event-stream');
     res.setHeader('Cache-Control', 'no-cache');
@@ -59,22 +62,15 @@ app.post('/api/migrate', (req, res) => {
     }
     sendEvent('info', `Found script at: ${scriptPath}`);
     sendEvent('info', `Working directory: ${projectRoot}`);
-    // Prepare environment - support both access token and database password
+    // Prepare environment with access token for Supabase CLI
     const env = {
         ...process.env,
         SUPABASE_PROJECT_ID: projectId,
+        SUPABASE_ACCESS_TOKEN: accessToken,
         // Ensure PATH includes common locations for supabase CLI
         PATH: `${process.env.PATH}:/usr/local/bin:/opt/homebrew/bin:${projectRoot}/node_modules/.bin`
     };
-    // Access token is preferred for non-interactive auth
-    if (accessToken) {
-        env.SUPABASE_ACCESS_TOKEN = accessToken;
-        sendEvent('info', 'Using access token for authentication');
-    }
-    if (dbPassword) {
-        env.SUPABASE_DB_PASSWORD = dbPassword;
-        sendEvent('info', 'Using database password for authentication');
-    }
+    sendEvent('info', 'Using access token for authentication');
     // Track process state
     let processCompleted = false;
     // Spawn the migration script in its own process group

package/dist/api/services/MinerService.js

@@ -6,9 +6,9 @@ import { CONFIG } from '../config/index.js';
 import { ProcessingEventService } from './ProcessingEventService.js';
 import { UrlNormalizer } from '../utils/UrlNormalizer.js';
 export class MinerService {
-    // Timestamp conversion constants
-    static WEBKIT_EPOCH_OFFSET_MS = 11644473600000; // Milliseconds between 1601-01-01 (WebKit epoch) and 1970-01-01 (Unix epoch)
-    static SAFARI_EPOCH_OFFSET_SEC = 978307200; // Seconds between 1970-01-01 (Unix epoch) and 2001-01-01 (Safari epoch)
+    // Timestamp conversion constants (using BigInt for precision)
+    static WEBKIT_EPOCH_OFFSET_MS = 11644473600000n; // Milliseconds between 1601-01-01 and 1970-01-01
+    static SAFARI_EPOCH_OFFSET_SEC = 978307200; // Seconds between 1970-01-01 and 2001-01-01
     static SANITY_CHECK_THRESHOLD = 3000000000000; // Timestamp threshold for Year ~2065, used to detect invalid/raw format timestamps
     processingEvents = ProcessingEventService.getInstance();
     debugMode = false;
@@ -155,20 +155,24 @@ export class MinerService {
                         SELECT url, title, visit_count, last_visit_date as last_visit_time
                         FROM moz_places
                         WHERE last_visit_date > ? AND url LIKE 'http%'
-                        ORDER BY last_visit_date DESC
+                        ORDER BY last_visit_date ASC
                         LIMIT ?
                     `;
                 }
                 else {
                     // Chrome, Edge, Brave, Arc, Safari (usually)
                     if (source.browser === 'safari') {
-                        // Safari uses Core Data timestamp (seconds since 2001-01-01)
-                        // Not fully implemented yet, but keeping placeholder
+                        // Safari: Join visits and items
                         query = `
-                            SELECT url, title, visit_count, last_visit_time 
-                            FROM history_items 
-                            WHERE last_visit_time > ? 
-                            ORDER BY last_visit_time DESC 
+                            SELECT 
+                                i.url, 
+                                i.title, 
+                                i.visit_count, 
+                                v.visit_time as last_visit_time 
+                            FROM history_visits v
+                            JOIN history_items i ON v.history_item = i.id
+                            WHERE v.visit_time > ? 
+                            ORDER BY v.visit_time ASC 
                             LIMIT ?
                         `;
                     }
@@ -177,7 +181,7 @@ export class MinerService {
                             SELECT url, title, visit_count, last_visit_time 
                             FROM urls 
                             WHERE last_visit_time > ?
-                            ORDER BY last_visit_time DESC 
+                            ORDER BY last_visit_time ASC 
                             LIMIT ?
                         `;
                     }
@@ -239,8 +243,10 @@ export class MinerService {
             if (skippedDuplicates > 0 || skippedNonContent > 0 || skippedBlacklist > 0) {
                 this.debug(`URL Filtering: ${skippedDuplicates} duplicates, ${skippedNonContent} non-content, ${skippedBlacklist} blacklisted`);
             }
-            if (entries.length > 0) {
-                const newestTime = Math.max(...entries.map(e => e.last_visit_time));
+            if (rows.length > 0) {
+                // Since we order ASC, the last row is the latest time in this batch
+                const lastRow = rows[rows.length - 1];
+                const newestTime = this.toUnixMs(lastRow.last_visit_time, source.browser);
                 await this.saveCheckpoint(source.path, newestTime, supabase, userId);
                 // Also update last_sync_checkpoint in settings for global tracking
                 if (userId && settings) {
@@ -261,29 +267,36 @@ export class MinerService {
             return Date.now();
         if (browser === 'firefox') {
             // Firefox: Microseconds -> Milliseconds
-            return Math.floor(timestamp / 1000);
+            return Number(BigInt(timestamp) / 1000n);
         }
         else if (browser === 'safari') {
-            // Safari: Seconds since 2001-01-01 -> Unix Ms
-            return Math.floor((timestamp + MinerService.SAFARI_EPOCH_OFFSET_SEC) * 1000);
+            // Safari: Seconds (float) since 2001 -> Unix Ms
+            // Safari uses floats (CFAbsoluteTime), keep as Number
+            return Math.floor((Number(timestamp) + MinerService.SAFARI_EPOCH_OFFSET_SEC) * 1000);
         }
         else {
-            // Chrome/Webkit: Microseconds since 1601-01-01 -> Unix Ms
-            return Math.floor((timestamp / 1000) - MinerService.WEBKIT_EPOCH_OFFSET_MS);
+            // Chrome/Webkit: Microseconds since 1601 -> Unix Ms
+            const ts = BigInt(timestamp);
+            const microDiff = ts - (MinerService.WEBKIT_EPOCH_OFFSET_MS * 1000n);
+            return Number(microDiff / 1000n);
         }
     }
     fromUnixMs(unixMs, browser) {
         if (!unixMs)
-            return 0; // Default to beginning of time
+            return 0;
         if (browser === 'firefox') {
-            return unixMs * 1000;
+            // Firefox: Milliseconds -> Microseconds (BigInt)
+            return BigInt(unixMs) * 1000n;
         }
         else if (browser === 'safari') {
+            // Safari: Unix Ms -> Seconds since 2001 (Float)
             return (unixMs / 1000) - MinerService.SAFARI_EPOCH_OFFSET_SEC;
         }
         else {
-            // Chrome/Webkit
-            return (unixMs + MinerService.WEBKIT_EPOCH_OFFSET_MS) * 1000;
+            // Chrome/Webkit: Unix Ms -> Microseconds since 1601 (BigInt)
+            // (UnixMs + OffsetMs) * 1000 = Microseconds
+            const unixBig = BigInt(unixMs);
+            return (unixBig + MinerService.WEBKIT_EPOCH_OFFSET_MS) * 1000n;
         }
     }
     async getCheckpoint(browser, supabase) {