liferewind 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Richard Wang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,219 @@
+# liferewind
+
+[![npm version](https://img.shields.io/npm/v/liferewind.svg)](https://www.npmjs.com/package/liferewind)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+AI-powered personal life review tool - collect your digital footprints from git, browser history, documents, and AI chatbots.
+
+## Installation
+
+```bash
+npm install -g liferewind
+```
+
+## Quick Start
+
+```bash
+# Interactive configuration wizard
+liferewind init
+
+# Start collecting
+liferewind start
+
+# Manual collection
+liferewind collect
+```
+
+## Data Sources
+
+| Type | Status | Description |
+|------|--------|-------------|
+| Git | ✅ Implemented | Git commit history |
+| Browser | ✅ Implemented | Browser history (Chrome, Safari, Arc, Dia, Comet) |
+| Filesystem | ✅ Implemented | Document change tracking (.md, .txt, .docx, .pdf, etc.) |
+| Chatbot | ✅ Implemented | Local AI chatbot history (ChatWise) |
+
+## Configuration
+
+Run `liferewind init` for interactive setup, or create config manually.
+
+Config file lookup order:
+1. `~/.liferewind/config.json` (primary user config)
+2. `~/.config/liferewind/collector.json`
+3. `~/.liferewind-collector.json`
+4. `./collector.config.json`
+
+Example configuration (see `collector.config.example.json`):
+
+```json
+{
+  "api": {
+    "baseUrl": "http://localhost:3000",
+    "apiKey": "your-api-key"
+  },
+  "sources": {
+    "git": {
+      "enabled": true,
+      "schedule": "daily",
+      "options": {
+        "scanPaths": ["~/Documents", "~/Projects"],
+        "sinceDays": 30
+      }
+    },
+    "browser": {
+      "enabled": true,
+      "schedule": "daily",
+      "options": {
+        "browsers": ["chrome", "safari", "arc"],
+        "excludeDomains": ["localhost", "127.0.0.1"],
+        "sinceDays": 7
+      }
+    },
+    "filesystem": {
+      "enabled": false,
+      "schedule": "daily",
+      "options": {
+        "watchPaths": ["~/Documents"],
+        "excludePatterns": ["**/node_modules/**", "**/.git/**"],
+        "fileTypes": [".md", ".txt", ".docx", ".pdf"],
+        "sinceDays": 7,
+        "includeContent": true
+      }
+    },
+    "chatbot": {
+      "enabled": false,
+      "schedule": "daily",
+      "options": {
+        "clients": ["chatwise"],
+        "sinceDays": 30,
+        "includeContent": true
+      }
+    }
+  },
+  "logging": {
+    "level": "info"
+  }
+}
+```
+
+**Note:** Git repositories are automatically excluded from filesystem scanning.
+
+Environment variables (fallback):
+```bash
+export LIFEREWIND_API_URL="http://localhost:3000"
+export LIFEREWIND_API_KEY="your-api-key"
+```
+
+## CLI Commands
+
+```bash
+# Configuration
+liferewind init              # Interactive setup wizard
+liferewind config show       # Display current config
+liferewind config edit       # Open config in editor
+liferewind config validate   # Validate config file
+liferewind config path       # Show config file locations
+
+# Collection
+liferewind start             # Start collector service
+liferewind start --run-once  # Collect once and exit
+liferewind collect           # Manual trigger (all sources)
+liferewind collect git       # Manual trigger (specific source)
+
+# Diagnostics
+liferewind status            # Show service status
+liferewind doctor            # Check environment issues
+```
+
+## Development
+
+```bash
+# Development with watch
+pnpm dev
+
+# Build
+pnpm build
+
+# Production
+pnpm start
+
+# PM2 daemon
+pnpm pm2:start
+pnpm pm2:stop
+pnpm pm2:logs
+```
+
+## Schedule Frequencies
+
+| Value | Cron | Description |
+|-------|------|-------------|
+| `hourly` | `0 * * * *` | Every hour |
+| `daily` | `0 9 * * *` | Daily at 9:00 |
+| `weekly` | `0 9 * * 1` | Monday at 9:00 |
+| `monthly` | `0 9 1 * *` | 1st of month at 9:00 |
+| `manual` | - | Manual trigger only |
+
+## Adding a Data Source
+
+1. Create a new directory under `src/sources/`
+2. Extend the `DataSource<TOptions>` base class
+3. Register in `src/sources/index.ts`
+
+```typescript
+// src/sources/my-source/index.ts
+import { DataSource } from '../base.js';
+
+interface MySourceOptions {
+  // ...
+}
+
+export class MySource extends DataSource<MySourceOptions> {
+  readonly type = 'my-source' as const;
+  readonly name = 'My Source';
+
+  async validate(): Promise<boolean> {
+    // Validate environment requirements
+  }
+
+  async collect(): Promise<CollectionResult> {
+    // Collect data
+  }
+}
+
+// src/sources/index.ts
+sourceRegistry.register('my-source', (config, ctx) => new MySource(config, ctx));
+```
+
+## Project Structure
+
+```
+src/
+├── index.ts          # Library exports
+├── cli/              # CLI commands
+│   ├── index.ts      # CLI entry point
+│   ├── commands/     # Command implementations
+│   ├── detect/       # Environment detection
+│   └── utils/        # CLI utilities
+├── core/
+│   ├── types.ts      # Type definitions
+│   ├── collector.ts  # Main orchestrator
+│   └── scheduler.ts  # Cron scheduler
+├── sources/
+│   ├── base.ts       # DataSource base class
+│   ├── registry.ts   # Source registry
+│   ├── index.ts      # Source registration
+│   ├── git/          # Git data source
+│   ├── browser/      # Browser history (multi-browser)
+│   ├── filesystem/   # Filesystem changes
+│   └── chatbot/      # Chatbot history (multi-client)
+├── api/
+│   └── client.ts     # API client
+├── config/
+│   ├── schema.ts     # Config schema (Zod)
+│   ├── loader.ts     # Config loader
+│   ├── writer.ts     # Config writer
+│   └── paths.ts      # Config path constants
+└── utils/
+    ├── logger.ts     # Logger
+    └── retry.ts      # Retry with backoff
+```

package/dist/api/client.d.ts

@@ -0,0 +1,31 @@
+import { z } from 'zod';
+import type { CollectionResult } from '../core/types.js';
+import type { Logger } from '../utils/logger.js';
+export interface ApiClientConfig {
+    baseUrl: string;
+    apiKey: string;
+    timeout: number;
+    retryAttempts: number;
+}
+declare const pushResponseSchema: z.ZodObject<{
+    success: z.ZodLiteral<true>;
+    data: z.ZodObject<{
+        itemsReceived: z.ZodNumber;
+        itemsInserted: z.ZodNumber;
+        requestId: z.ZodString;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export type PushResponse = z.infer<typeof pushResponseSchema>;
+export declare class ApiClient {
+    private static readonly BATCH_SIZE;
+    private config;
+    private logger;
+    constructor(config: ApiClientConfig, logger: Logger);
+    pushData(result: CollectionResult): Promise<PushResponse>;
+    healthCheck(): Promise<boolean>;
+    private pushInBatches;
+    private pushBatch;
+    private splitIntoBatches;
+}
+export {};
+//# sourceMappingURL=client.d.ts.map

package/dist/api/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/api/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAGjD,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,QAAA,MAAM,kBAAkB;;;;;;;iBAOtB,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAO9D,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAE1C,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM;IAK7C,QAAQ,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC;IAQzD,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;YAWvB,aAAa;YA4Db,SAAS;IAuCvB,OAAO,CAAC,gBAAgB;CAOzB"}

package/dist/api/client.js

@@ -0,0 +1,115 @@
+import { z } from 'zod';
+import { retry } from '../utils/retry.js';
+const pushResponseSchema = z.object({
+    success: z.literal(true),
+    data: z.object({
+        itemsReceived: z.number(),
+        itemsInserted: z.number(),
+        requestId: z.string(),
+    }),
+});
+export class ApiClient {
+    static BATCH_SIZE = 1000;
+    config;
+    logger;
+    constructor(config, logger) {
+        this.config = config;
+        this.logger = logger;
+    }
+    async pushData(result) {
+        if (result.items.length <= ApiClient.BATCH_SIZE) {
+            return this.pushBatch(result);
+        }
+        return this.pushInBatches(result);
+    }
+    async healthCheck() {
+        try {
+            const response = await fetch(`${this.config.baseUrl}/api/health`, {
+                signal: AbortSignal.timeout(5000),
+            });
+            return response.ok;
+        }
+        catch {
+            return false;
+        }
+    }
+    async pushInBatches(result) {
+        const batches = this.splitIntoBatches(result.items, ApiClient.BATCH_SIZE);
+        this.logger.info(`Splitting ${result.items.length} items into ${batches.length} batches (${ApiClient.BATCH_SIZE} items/batch)`);
+        const batchResults = [];
+        for (const [index, batch] of batches.entries()) {
+            const batchNumber = index + 1;
+            this.logger.info(`Pushing batch ${batchNumber}/${batches.length} (${batch.length} items)`);
+            try {
+                const response = await this.pushBatch({
+                    ...result,
+                    items: batch,
+                    itemsCollected: batch.length,
+                });
+                batchResults.push({
+                    requestId: response.data.requestId,
+                    itemsInserted: response.data.itemsInserted,
+                });
+                this.logger.info(`Batch ${batchNumber}/${batches.length} completed: ` +
+                    `requestId=${response.data.requestId}, inserted=${response.data.itemsInserted}/${batch.length}`);
+            }
+            catch (error) {
+                const successfulBatches = batchResults.length;
+                const totalInserted = batchResults.reduce((sum, r) => sum + r.itemsInserted, 0);
+                this.logger.error(`Batch ${batchNumber}/${batches.length} failed. ` +
+                    `${successfulBatches} batches succeeded (${totalInserted} items inserted).`, error);
+                throw new Error(`Batch ${batchNumber}/${batches.length} failed after ${successfulBatches} successful batches ` +
+                    `(${totalInserted} items inserted). ${error instanceof Error ? error.message : 'Unknown error'}`);
+            }
+        }
+        const totalInserted = batchResults.reduce((sum, r) => sum + r.itemsInserted, 0);
+        const lastRequestId = batchResults[batchResults.length - 1]?.requestId ?? 'unknown';
+        return {
+            success: true,
+            data: {
+                itemsReceived: result.items.length,
+                itemsInserted: totalInserted,
+                requestId: lastRequestId,
+            },
+        };
+    }
+    async pushBatch(result) {
+        const url = `${this.config.baseUrl}/api/collector/ingest`;
+        return retry(async () => {
+            const response = await fetch(url, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${this.config.apiKey}`,
+                    'X-Source-Type': result.sourceType,
+                },
+                body: JSON.stringify({
+                    sourceType: result.sourceType,
+                    collectedAt: result.collectedAt.toISOString(),
+                    items: result.items,
+                }),
+                signal: AbortSignal.timeout(this.config.timeout),
+            });
+            if (!response.ok) {
+                const errorText = await response.text();
+                throw new Error(`API error ${response.status}: ${errorText}`);
+            }
+            const json = await response.json();
+            return pushResponseSchema.parse(json);
+        }, {
+            attempts: this.config.retryAttempts,
+            delay: 1000,
+            backoff: 2,
+            onRetry: (error, attempt) => {
+                this.logger.warn(`API push retry ${attempt}/${this.config.retryAttempts}`, error);
+            },
+        });
+    }
+    splitIntoBatches(items, batchSize) {
+        const batches = [];
+        for (let i = 0; i < items.length; i += batchSize) {
+            batches.push(items.slice(i, i + batchSize));
+        }
+        return batches;
+    }
+}

package/dist/cli/commands/collect.d.ts

@@ -0,0 +1,3 @@
+import { Command } from 'commander';
+export declare const collectCommand: Command;
+//# sourceMappingURL=collect.d.ts.map

package/dist/cli/commands/collect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"collect.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/collect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,eAAO,MAAM,cAAc,SA0DvB,CAAC"}

package/dist/cli/commands/collect.js

@@ -0,0 +1,62 @@
+import { Command } from 'commander';
+import ora from 'ora';
+import { Collector } from '../../core/collector.js';
+import { loadConfig } from '../../config/loader.js';
+import { createLogger } from '../../utils/logger.js';
+import { registerBuiltinSources } from '../../sources/index.js';
+import { printError, printSuccess } from '../utils/output.js';
+import { SOURCE_TYPES } from '../../core/types.js';
+export const collectCommand = new Command('collect')
+    .description('Manually trigger data collection')
+    .argument('[source]', 'specific source to collect (git, browser, filesystem, chatbot)')
+    .action(async (source, _options, cmd) => {
+    const globalOpts = cmd.optsWithGlobals();
+    // Validate source argument
+    if (source && !SOURCE_TYPES.includes(source)) {
+        printError(`Invalid source: ${source}`);
+        console.log(`Valid sources: ${SOURCE_TYPES.join(', ')}`);
+        process.exit(1);
+    }
+    try {
+        const config = loadConfig(globalOpts.config);
+        // Apply global log level overrides
+        let logLevel = config.logging.level;
+        if (globalOpts.verbose)
+            logLevel = 'debug';
+        if (globalOpts.quiet)
+            logLevel = 'error';
+        const logger = createLogger({ level: logLevel });
+        registerBuiltinSources();
+        const collector = new Collector(config, logger);
+        await collector.validateSources();
+        const enabledSources = collector.getEnabledSources();
+        if (enabledSources.length === 0) {
+            printError('No data sources enabled or validated.');
+            process.exit(1);
+        }
+        const spinner = ora();
+        if (source) {
+            // Collect specific source
+            if (!enabledSources.includes(source)) {
+                printError(`Source '${source}' is not enabled or not validated.`);
+                process.exit(1);
+            }
+            spinner.start(`Collecting from ${source}...`);
+            await collector.triggerCollection(source);
+            spinner.succeed(`Collection from ${source} complete`);
+        }
+        else {
+            // Collect all enabled sources
+            for (const src of enabledSources) {
+                spinner.start(`Collecting from ${src}...`);
+                await collector.triggerCollection(src);
+                spinner.succeed(`Collection from ${src} complete`);
+            }
+        }
+        printSuccess('All collections complete.');
+    }
+    catch (error) {
+        printError(`Collection failed: ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    }
+});

package/dist/cli/commands/config.d.ts

@@ -0,0 +1,3 @@
+import { Command } from 'commander';
+export declare const configCommand: Command;
+//# sourceMappingURL=config.d.ts.map

package/dist/cli/commands/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,eAAO,MAAM,aAAa,SAA4D,CAAC"}

package/dist/cli/commands/config.js

@@ -0,0 +1,112 @@
+import { Command } from 'commander';
+import { execSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import pc from 'picocolors';
+import { loadConfig, findConfigPath } from '../../config/loader.js';
+import { getUserConfigPath, getAllConfigPaths } from '../../config/paths.js';
+import { printError, printSuccess, printInfo, printDim } from '../utils/output.js';
+export const configCommand = new Command('config').description('Manage configuration');
+function printConfigSummary(config, revealSecrets = false) {
+    console.log(pc.bold('\nAPI Configuration'));
+    console.log(`  Base URL: ${config.api.baseUrl}`);
+    const apiKey = revealSecrets
+        ? config.api.apiKey
+        : config.api.apiKey.length > 12
+            ? config.api.apiKey.slice(0, 8) + '...' + config.api.apiKey.slice(-4)
+            : '***';
+    console.log(`  API Key:  ${apiKey}`);
+    console.log(`  Timeout:  ${config.api.timeout}ms`);
+    console.log(`  Retries:  ${config.api.retryAttempts}`);
+    console.log(pc.bold('\nData Sources'));
+    const sources = ['git', 'browser', 'filesystem', 'chatbot'];
+    for (const name of sources) {
+        const src = config.sources[name];
+        const status = src.enabled ? pc.green('enabled') : pc.dim('disabled');
+        const schedule = src.enabled ? ` (${src.schedule})` : '';
+        console.log(`  ${name}: ${status}${schedule}`);
+    }
+    console.log(pc.bold('\nLogging'));
+    console.log(`  Level: ${config.logging.level}`);
+}
+configCommand
+    .command('show')
+    .description('Display current configuration')
+    .option('--json', 'output as JSON')
+    .option('--reveal-secrets', 'show API keys (hidden by default)')
+    .action((options, cmd) => {
+    const globalOpts = cmd.optsWithGlobals();
+    try {
+        const config = loadConfig(globalOpts.config);
+        const configPath = findConfigPath(globalOpts.config);
+        if (configPath) {
+            printDim(`Loaded from: ${configPath}`);
+        }
+        // Helper to mask API key
+        const maskApiKey = (key) => key.length > 12 ? key.slice(0, 8) + '...' + key.slice(-4) : '***';
+        if (options.json) {
+            const output = {
+                ...config,
+                api: {
+                    ...config.api,
+                    apiKey: options.revealSecrets ? config.api.apiKey : maskApiKey(config.api.apiKey),
+                },
+            };
+            console.log(JSON.stringify(output, null, 2));
+        }
+        else {
+            printConfigSummary(config, options.revealSecrets);
+        }
+    }
+    catch (error) {
+        printError(`Failed to load config: ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    }
+});
+configCommand
+    .command('path')
+    .description('Show configuration file paths')
+    .action(() => {
+    const paths = getAllConfigPaths();
+    console.log('Configuration file search order:');
+    paths.forEach((p, i) => {
+        const exists = existsSync(p);
+        const status = exists ? '(exists)' : '';
+        console.log(`  ${i + 1}. ${p} ${status}`);
+    });
+    console.log(`\nPrimary user config: ${getUserConfigPath()}`);
+});
+configCommand
+    .command('edit')
+    .description('Open configuration in default editor')
+    .action((_options, cmd) => {
+    const globalOpts = cmd.optsWithGlobals();
+    const configPath = findConfigPath(globalOpts.config);
+    if (!configPath) {
+        printError('No config file found.');
+        printInfo("Run 'liferewind init' to create one.");
+        process.exit(1);
+    }
+    const editor = process.env['EDITOR'] || process.env['VISUAL'] || (process.platform === 'darwin' ? 'open' : 'vi');
+    try {
+        execSync(`${editor} "${configPath}"`, { stdio: 'inherit' });
+    }
+    catch (error) {
+        printError(`Failed to open editor: ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    }
+});
+configCommand
+    .command('validate')
+    .description('Validate configuration file')
+    .action((_options, cmd) => {
+    const globalOpts = cmd.optsWithGlobals();
+    try {
+        loadConfig(globalOpts.config);
+        const configPath = findConfigPath(globalOpts.config);
+        printSuccess(`Configuration is valid: ${configPath}`);
+    }
+    catch (error) {
+        printError(`Configuration invalid: ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    }
+});

package/dist/cli/commands/doctor.d.ts

@@ -0,0 +1,3 @@
+import { Command } from 'commander';
+export declare const doctorCommand: Command;
+//# sourceMappingURL=doctor.d.ts.map

package/dist/cli/commands/doctor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoHpC,eAAO,MAAM,aAAa,SAiDtB,CAAC"}