@bobfrankston/gcards 0.1.2 → 0.1.5

package/gcards.ts

@@ -5,102 +5,58 @@
  */
 
 import fs from 'fs';
+import fp from 'fs/promises';
 import path from 'path';
-import { parseArgs, showHelp } from './cli.ts';
+import crypto from 'crypto';
+import { parseArgs, showUsage, showHelp } from './glib/parsecli.ts';
 import { authenticateOAuth } from '../../../projects/oauth/oauthsupport/index.ts';
-import type { GooglePerson, GoogleConnectionsResponse } from './types.ts';
-
-const APP_DIR = import.meta.dirname;
-const DATA_DIR = path.join(APP_DIR, 'data');
-const CREDENTIALS_FILE = path.join(APP_DIR, 'credentials.json');
-const CONFIG_FILE = path.join(DATA_DIR, 'config.json');
+import type { GooglePerson, GoogleConnectionsResponse } from './glib/types.ts';
+import { GCARDS_GUID_KEY, extractGuids } from './glib/gctypes.ts';
+import type { ContactIndex, IndexEntry, DeletedEntry, DeleteQueue, DeleteQueueEntry, PushStatus, PendingChange, UserPaths } from './glib/gctypes.ts';
+import { DATA_DIR, CREDENTIALS_FILE, loadConfig, saveConfig, getUserPaths, ensureUserDir, loadIndex, normalizeUser, getAllUsers, resolveUser, FileLogger } from './glib/gutils.ts';
 
 const PEOPLE_API_BASE = 'https://people.googleapis.com/v1';
 const CONTACTS_SCOPE_READ = 'https://www.googleapis.com/auth/contacts.readonly';
 const CONTACTS_SCOPE_WRITE = 'https://www.googleapis.com/auth/contacts';
 
-interface AppConfig {
-    lastUser?: string;
+function ts(): string {
+    const now = new Date();
+    return `[${now.toTimeString().slice(0, 8)}]`;
 }
 
-interface UserPaths {
-    userDir: string;
-    contactsDir: string;
-    deletedDir: string;
-    indexFile: string;
-    statusFile: string;
-    syncTokenFile: string;
-    tokenFile: string;
-    tokenWriteFile: string;
-}
+let escapePressed = false;
 
-function loadConfig(): AppConfig {
-    if (fs.existsSync(CONFIG_FILE)) {
-        return JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf-8'));
-    }
-    return {};
-}
+function setupEscapeHandler(): void {
+    // Handle Ctrl+C via SIGINT (works without raw mode)
+    process.on('SIGINT', () => {
+        escapePressed = true;
+        console.log('\n\nCtrl+C pressed - stopping after current operation...');
+    });
 
-function saveConfig(config: AppConfig): void {
-    if (!fs.existsSync(DATA_DIR)) {
-        fs.mkdirSync(DATA_DIR, { recursive: true });
+    // Handle ESC key via raw mode (optional, for graceful stop)
+    if (process.stdin.isTTY) {
+        process.stdin.setRawMode(true);
+        process.stdin.resume();
+        process.stdin.on('data', (key) => {
+            if (key[0] === 27) {  // ESC key
+                escapePressed = true;
+                console.log('\n\nESC pressed - stopping after current operation...');
+            } else if (key[0] === 3) {  // Ctrl+C in raw mode
+                escapePressed = true;
+                console.log('\n\nCtrl+C pressed - stopping after current operation...');
+            }
+        });
     }
-    fs.writeFileSync(CONFIG_FILE, JSON.stringify(config, null, 2));
 }
 
-/** Normalize Gmail username: lowercase, remove dots, strip +suffix and @domain */
-function normalizeUser(user: string): string {
-    return user.toLowerCase().split(/[+@]/)[0].replace(/\./g, '');
-}
-
-function getUserPaths(user: string): UserPaths {
-    const userDir = path.join(DATA_DIR, user);
-    return {
-        userDir,
-        contactsDir: path.join(userDir, 'contacts'),
-        deletedDir: path.join(userDir, 'deleted'),
-        indexFile: path.join(userDir, 'index.json'),
-        statusFile: path.join(userDir, 'status.json'),
-        syncTokenFile: path.join(userDir, 'sync-token.json'),
-        tokenFile: path.join(userDir, 'token.json'),
-        tokenWriteFile: path.join(userDir, 'token-write.json')
-    };
-}
-
-function ensureUserDir(user: string): void {
-    const paths = getUserPaths(user);
-    if (!fs.existsSync(paths.userDir)) {
-        fs.mkdirSync(paths.userDir, { recursive: true });
+function cleanupEscapeHandler(): void {
+    if (process.stdin.isTTY) {
+        process.stdin.setRawMode(false);
+        process.stdin.pause();
     }
 }
 
-interface ContactIndex {
-    contacts: Record<string, IndexEntry>;  /** resourceName -> entry */
-    lastSync: string;
-    syncToken?: string;
-}
-
-interface IndexEntry {
-    resourceName: string;
-    displayName: string;
-    etag?: string;
-    deleted: boolean;
-    deletedAt?: string;
-    updatedAt: string;
-}
-
-interface PushStatus {
-    lastPush: string;  /** ISO timestamp of last push */
-}
-
-interface PendingChange {
-    type: 'update' | 'delete';
-    resourceName: string;
-    displayName: string;
-    filePath: string;
-}
-
-async function getAccessToken(user: string, writeAccess = false): Promise<string> {
+async function getAccessToken(user: string, writeAccess = false, forceRefresh = false): Promise<string> {
     if (!fs.existsSync(CREDENTIALS_FILE)) {
         throw new Error(`Credentials file not found: ${CREDENTIALS_FILE}`);
     }
@@ -110,6 +66,13 @@ async function getAccessToken(user: string, writeAccess = false): Promise<string
 
     const scope = writeAccess ? CONTACTS_SCOPE_WRITE : CONTACTS_SCOPE_READ;
     const tokenFileName = writeAccess ? 'token-write.json' : 'token.json';
+    const tokenFilePath = path.join(paths.userDir, tokenFileName);
+
+    // Delete cached token to force re-auth
+    if (forceRefresh && fs.existsSync(tokenFilePath)) {
+        fs.unlinkSync(tokenFilePath);
+        console.log(`${ts()} Token expired, refreshing...`);
+    }
 
     const token = await authenticateOAuth(CREDENTIALS_FILE, {
         scope,
@@ -126,18 +89,122 @@ async function getAccessToken(user: string, writeAccess = false): Promise<string
     return token.access_token;
 }
 
-function loadIndex(paths: UserPaths): ContactIndex {
-    if (fs.existsSync(paths.indexFile)) {
-        return JSON.parse(fs.readFileSync(paths.indexFile, 'utf-8'));
-    }
-    return { contacts: {}, lastSync: '' };
+// Mutable token holder for refresh during long operations
+let currentAccessToken = '';
+let currentUser = '';
+
+async function refreshAccessToken(): Promise<string> {
+    currentAccessToken = await getAccessToken(currentUser, true, true);
+    return currentAccessToken;
 }
 
 function saveIndex(paths: UserPaths, index: ContactIndex): void {
+    // Sort contacts by displayName for convenience
+    const sortedContacts: Record<string, IndexEntry> = {};
+    const entries = Object.entries(index.contacts);
+    entries.sort((a, b) => a[1].displayName.localeCompare(b[1].displayName));
+    for (const [key, value] of entries) {
+        sortedContacts[key] = value;
+    }
+    index.contacts = sortedContacts;
     fs.writeFileSync(paths.indexFile, JSON.stringify(index, null, 2));
 }
 
-function loadSyncToken(paths: UserPaths): string | null {
+/** Deleted contacts index (separate from active index) */
+interface DeletedIndex {
+    deleted: Record<string, DeletedEntry>;
+}
+
+function loadDeleted(paths: UserPaths): DeletedIndex {
+    if (fs.existsSync(paths.deletedFile)) {
+        return JSON.parse(fs.readFileSync(paths.deletedFile, 'utf-8'));
+    }
+    return { deleted: {} };
+}
+
+function saveDeleted(paths: UserPaths, deletedIndex: DeletedIndex): void {
+    // Sort by displayName
+    const sorted: Record<string, DeletedEntry> = {};
+    const entries = Object.entries(deletedIndex.deleted);
+    entries.sort((a, b) => a[1].displayName.localeCompare(b[1].displayName));
+    for (const [key, value] of entries) {
+        sorted[key] = value;
+    }
+    deletedIndex.deleted = sorted;
+    fs.writeFileSync(paths.deletedFile, JSON.stringify(deletedIndex, null, 2));
+}
+
+interface SavedPhoto {
+    url: string;
+    sourceType: 'CONTACT' | 'PROFILE';
+}
+
+interface PhotoEntry {
+    displayName: string;
+    photos: SavedPhoto[];
+    deletedAt: string;
+}
+
+interface PhotosIndex {
+    photos: Record<string, PhotoEntry>;  // resourceName -> PhotoEntry
+}
+
+function loadPhotos(paths: UserPaths): PhotosIndex {
+    if (fs.existsSync(paths.photosFile)) {
+        return JSON.parse(fs.readFileSync(paths.photosFile, 'utf-8'));
+    }
+    return { photos: {} };
+}
+
+function savePhotos(paths: UserPaths, photosIndex: PhotosIndex): void {
+    // Sort by displayName
+    const sorted: Record<string, PhotoEntry> = {};
+    const entries = Object.entries(photosIndex.photos);
+    entries.sort((a, b) => a[1].displayName.localeCompare(b[1].displayName));
+    for (const [key, value] of entries) {
+        sorted[key] = value;
+    }
+    photosIndex.photos = sorted;
+    fs.writeFileSync(paths.photosFile, JSON.stringify(photosIndex, null, 2));
+}
+
+function loadDeleteQueue(paths: UserPaths): DeleteQueue {
+    if (fs.existsSync(paths.deleteQueueFile)) {
+        return JSON.parse(fs.readFileSync(paths.deleteQueueFile, 'utf-8'));
+    }
+    return { updatedAt: '', entries: [] };
+}
+
+function saveDeleteQueue(paths: UserPaths, queue: DeleteQueue): void {
+    queue.updatedAt = new Date().toISOString();
+    fs.writeFileSync(paths.deleteQueueFile, JSON.stringify(queue, null, 2));
+}
+
+function extractNonDefaultPhotos(contact: GooglePerson): SavedPhoto[] {
+    if (!contact.photos) return [];
+    return contact.photos
+        .filter(p => !p.default)
+        .map(p => ({
+            url: p.url,
+            sourceType: (p.metadata?.source?.type || 'CONTACT') as 'CONTACT' | 'PROFILE'
+        }));
+}
+
+function hasNonDefaultPhoto(contact: GooglePerson): boolean {
+    return contact.photos?.some(p => !p.default) || false;
+}
+
+function isStarred(contact: GooglePerson): boolean {
+    return contact.memberships?.some(m =>
+        m.contactGroupMembership?.contactGroupId === 'starred'
+    ) || false;
+}
+
+function generateGuid(): string {
+    return crypto.randomUUID();
+}
+
+function loadSyncToken(paths: UserPaths): string {
     if (fs.existsSync(paths.syncTokenFile)) {
         const data = JSON.parse(fs.readFileSync(paths.syncTokenFile, 'utf-8'));
         return data.syncToken || null;
@@ -163,7 +230,7 @@ async function fetchContactsWithRetry(
     const BASE_DELAY = 5000;  /** 5 seconds base delay */
 
     const params = new URLSearchParams({
-        personFields: 'names,emailAddresses,phoneNumbers,addresses,organizations,biographies,urls,birthdays,photos,memberships,metadata',
+        personFields: 'names,emailAddresses,phoneNumbers,addresses,organizations,biographies,urls,birthdays,photos,memberships,metadata,fileAses',
         pageSize: '100'  /** Smaller page size to avoid quota issues */
     });
 
@@ -230,8 +297,39 @@ async function syncContacts(user: string, options: { full: boolean; verbose: boo
     const accessToken = await getAccessToken(user);
 
     const index = loadIndex(paths);
+    const deletedIndex = loadDeleted(paths);
+    const photosIndex = loadPhotos(paths);
     let syncToken = options.full ? null : loadSyncToken(paths);
 
+    // Migrate any old deleted entries from index.json to deleted.json
+    // Also clean up old fields (deleted: false, etag)
+    let migrated = 0;
+    let cleaned = 0;
+    for (const [rn, entry] of Object.entries(index.contacts)) {
+        const oldEntry = entry as any;
+        if (oldEntry.deleted === true) {
+            deletedIndex.deleted[rn] = { ...entry, deletedAt: oldEntry.deletedAt || new Date().toISOString() };
+            delete index.contacts[rn];
+            migrated++;
+        } else {
+            // Clean up old fields
+            if ('deleted' in oldEntry) {
+                delete oldEntry.deleted;
+                cleaned++;
+            }
+            if ('etag' in oldEntry) {
+                delete oldEntry.etag;
+                cleaned++;
+            }
+        }
+    }
+    if (migrated > 0) {
+        console.log(`Migrated ${migrated} deleted entries to deleted.json`);
+    }
+    if (cleaned > 0) {
+        console.log(`Cleaned ${cleaned} old fields from index entries`);
+    }
+
     if (syncToken) {
         console.log('Performing incremental sync...');
     } else {
@@ -243,6 +341,7 @@ async function syncContacts(user: string, options: { full: boolean; verbose: boo
     let added = 0;
     let updated = 0;
     let deleted = 0;
+    let conflicts = 0;
     let pageNum = 0;
 
     do {
@@ -257,23 +356,84 @@ async function syncContacts(user: string, options: { full: boolean; verbose: boo
                 const displayName = person.names?.[0]?.displayName || 'Unknown';
 
                 if (isDeleted) {
-                    if (index.contacts[person.resourceName]) {
-                        index.contacts[person.resourceName].deleted = true;
-                        index.contacts[person.resourceName].deletedAt = new Date().toISOString();
-                        deleteContactFile(paths, person.resourceName);
+                    const entry = index.contacts[person.resourceName];
+                    if (entry) {
+                        const id = person.resourceName.replace('people/', '');
+                        const contactFile = path.join(paths.contactsDir, `${id}.json`);
+
+                        // Save photos before deleting
+                        if (fs.existsSync(contactFile)) {
+                            try {
+                                const contact = JSON.parse(fs.readFileSync(contactFile, 'utf-8')) as GooglePerson;
+                                const photos = extractNonDefaultPhotos(contact);
+                                if (photos.length > 0) {
+                                    photosIndex.photos[person.resourceName] = {
+                                        displayName: entry.displayName,
+                                        photos,
+                                        deletedAt: new Date().toISOString()
+                                    };
+                                }
+                            } catch { /* ignore read errors */ }
+                        }
+
+                        // Move to deleted.json
+                        const deletedEntry = { ...entry, deletedAt: new Date().toISOString(), _delete: 'server' };
+                        deletedIndex.deleted[person.resourceName] = deletedEntry;
+                        delete index.contacts[person.resourceName];
+
+                        // Backup contact file to deleted/ folder
+                        if (fs.existsSync(contactFile)) {
+                            if (!fs.existsSync(paths.deletedDir)) {
+                                fs.mkdirSync(paths.deletedDir, { recursive: true });
+                            }
+                            const contactData = JSON.parse(fs.readFileSync(contactFile, 'utf-8'));
+                            contactData._deletedAt = new Date().toISOString();
+                            contactData._delete = 'server';
+                            const destPath = path.join(paths.deletedDir, `${id}.json`);
+                            fs.writeFileSync(destPath, JSON.stringify(contactData, null, 2));
+                            fs.unlinkSync(contactFile);
+                        }
+
                         deleted++;
                         if (options.verbose) console.log(`\n  Deleted: ${displayName}`);
                     }
                 } else {
                     const existed = !!index.contacts[person.resourceName];
+
+                    // Check for local conflict before overwriting
+                    const filePath = path.join(paths.contactsDir, `${person.resourceName.replace(/\//g, '_')}.json`);
+                    const indexEntry = index.contacts[person.resourceName];
+                    if (existed && fs.existsSync(filePath)) {
+                        const stat = fs.statSync(filePath);
+                        const modTime = stat.mtimeMs;
+                        const indexUpdatedAt = indexEntry?.updatedAt ? new Date(indexEntry.updatedAt).getTime() : 0;
+
+                        if (modTime > indexUpdatedAt + 1000) {
+                            // Local file was modified - conflict!
+                            console.log(`\n  CONFLICT: ${displayName} - local changes exist, skipping (delete local file and sync to get server version)`);
+                            conflicts++;
+                            continue;
+                        }
+                    }
+
                     saveContact(paths, person);
 
+                    // Extract GUIDs from userDefined fields
+                    const guids = extractGuids(person);
+                    const hasPhoto = hasNonDefaultPhoto(person);
+                    const starred = isStarred(person);
+
+                    // Preserve _delete flag from existing entry
+                    const existingDelete = index.contacts[person.resourceName]?._delete;
+
                     index.contacts[person.resourceName] = {
                         resourceName: person.resourceName,
                         displayName,
-                        etag: person.etag,
-                        deleted: false,
-                        updatedAt: new Date().toISOString()
+                        updatedAt: new Date().toISOString(),
+                        ...(hasPhoto && { hasPhoto }),
+                        ...(starred && { starred }),
+                        ...(guids.length > 0 && { guids }),
+                        ...(existingDelete && { _delete: existingDelete })
                     };
 
                     if (existed) {
@@ -298,21 +458,31 @@ async function syncContacts(user: string, options: { full: boolean; verbose: boo
         // Save index after each page in case of interruption
         index.lastSync = new Date().toISOString();
         saveIndex(paths, index);
+        saveDeleted(paths, deletedIndex);
+        savePhotos(paths, photosIndex);
 
         // Small delay between pages to be nice to the API
         if (pageToken) {
             await sleep(500);
         }
+
+        if (escapePressed) {
+            console.log('\n\nStopped by user. Progress saved.');
+            break;
+        }
     } while (pageToken);
 
-    const activeContacts = Object.values(index.contacts).filter(c => !c.deleted).length;
-    const tombstones = Object.values(index.contacts).filter(c => c.deleted).length;
+    const activeContacts = Object.keys(index.contacts).length;
+    const tombstones = Object.keys(deletedIndex.deleted).length;
 
     console.log(`\n\nSync complete:`);
     console.log(`  Processed: ${totalProcessed}`);
     console.log(`  Added: ${added}`);
     console.log(`  Updated: ${updated}`);
     console.log(`  Deleted: ${deleted}`);
+    if (conflicts > 0) {
+        console.log(`  Conflicts: ${conflicts} (delete local file and sync to resolve)`);
+    }
     console.log(`  Active contacts: ${activeContacts}`);
     console.log(`  Tombstones: ${tombstones}`);
 }
@@ -328,48 +498,96 @@ function saveStatus(paths: UserPaths, status: PushStatus): void {
     fs.writeFileSync(paths.statusFile, JSON.stringify(status, null, 2));
 }
 
-function findPendingChanges(paths: UserPaths): PendingChange[] {
-    const status = loadStatus(paths);
-    const lastPush = status.lastPush ? new Date(status.lastPush).getTime() : 0;
+interface PendingChangesResult {
+    changes: PendingChange[];
+    parseErrors: string[];
+}
+
+async function findPendingChanges(paths: UserPaths, logger: FileLogger, since?: string): Promise<PendingChangesResult> {
     const changes: PendingChange[] = [];
+    const parseErrors: string[] = [];
+    const index = loadIndex(paths);
+    // Use -since override if provided, otherwise use lastSync
+    const sinceTime = since ? new Date(since).getTime() : (index.lastSync ? new Date(index.lastSync).getTime() : 0);
+    if (since) {
+        console.log(`Using -since override: ${since}`);
+    }
+
+    // Check _delete.json for deletion requests
+    const deleteQueue = loadDeleteQueue(paths);
+    for (const entry of deleteQueue.entries) {
+        const filePath = path.join(paths.contactsDir, `${entry.resourceName.replace('people/', '')}.json`);
+        changes.push({
+            type: 'delete',
+            resourceName: entry.resourceName,
+            displayName: entry.displayName,
+            filePath,
+            _delete: entry._delete || 'queued'
+        });
+    }
 
-    // Check contacts/ for modified files and _deleted markers
+    // Check contacts/ for modified files and _delete markers
+    // Only check files modified after lastSync (fast filter using mtime)
     if (fs.existsSync(paths.contactsDir)) {
         const files = fs.readdirSync(paths.contactsDir).filter(f => f.endsWith('.json'));
+        let checked = 0;
+
         for (const file of files) {
             const filePath = path.join(paths.contactsDir, file);
-            const stat = fs.statSync(filePath);
+            const stat = await fp.stat(filePath);
             const modTime = stat.mtimeMs;
 
-            if (modTime > lastPush) {
-                const content = JSON.parse(fs.readFileSync(filePath, 'utf-8')) as GooglePerson & { _deleted?: boolean };
-                const displayName = content.names?.[0]?.displayName || 'Unknown';
+            // Skip files not modified since last sync (or -since override)
+            if (modTime <= sinceTime + 1000) continue;
+
+            checked++;
+            let content: GooglePerson & { _delete?: boolean };
+            try {
+                content = JSON.parse(await fp.readFile(filePath, 'utf-8'));
+            } catch (e: any) {
+                const errMsg = `[PARSE ERROR] ${file}: ${e.message}`;
+                parseErrors.push(errMsg);
+                logger.error(errMsg);
+                continue;
+            }
 
-                if (content._deleted) {
-                    changes.push({
-                        type: 'delete',
-                        resourceName: content.resourceName,
-                        displayName,
-                        filePath
-                    });
-                } else {
-                    changes.push({
-                        type: 'update',
-                        resourceName: content.resourceName,
-                        displayName,
-                        filePath
-                    });
-                }
+            const displayName = content.names?.[0]?.displayName || 'Unknown';
+
+            if (content._delete) {
+                changes.push({
+                    type: 'delete',
+                    resourceName: content.resourceName,
+                    displayName,
+                    filePath
+                });
+            } else {
+                changes.push({
+                    type: 'update',
+                    resourceName: content.resourceName,
+                    displayName,
+                    filePath
+                });
             }
         }
+        if (checked > 0) {
+            console.log(`${ts()} Found ${checked} modified contact(s)`);
+        }
     }
 
-    // Check deleted/ folder
-    if (fs.existsSync(paths.deletedDir)) {
-        const files = fs.readdirSync(paths.deletedDir).filter(f => f.endsWith('.json'));
+    // Check _delete/ folder for user deletion requests
+    if (fs.existsSync(paths.toDeleteDir)) {
+        const files = fs.readdirSync(paths.toDeleteDir).filter(f => f.endsWith('.json'));
         for (const file of files) {
-            const filePath = path.join(paths.deletedDir, file);
-            const content = JSON.parse(fs.readFileSync(filePath, 'utf-8')) as GooglePerson;
+            const filePath = path.join(paths.toDeleteDir, file);
+            let content: GooglePerson;
+            try {
+                content = JSON.parse(await fp.readFile(filePath, 'utf-8'));
+            } catch (e: any) {
+                const errMsg = `[PARSE ERROR] ${file}: ${e.message}`;
+                parseErrors.push(errMsg);
+                logger.error(errMsg);
+                continue;
+            }
             const displayName = content.names?.[0]?.displayName || 'Unknown';
 
             changes.push({
@@ -381,10 +599,41 @@ function findPendingChanges(paths: UserPaths): PendingChange[] {
         }
     }
 
-    return changes;
+    // Check _add/ folder for new contacts to create
+    if (fs.existsSync(paths.toAddDir)) {
+        const files = fs.readdirSync(paths.toAddDir).filter(f => f.endsWith('.json'));
+        for (const file of files) {
+            const filePath = path.join(paths.toAddDir, file);
+            let content: GooglePerson;
+            try {
+                content = JSON.parse(await fp.readFile(filePath, 'utf-8'));
+            } catch (e: any) {
+                const errMsg = `[PARSE ERROR] ${file}: ${e.message}`;
+                parseErrors.push(errMsg);
+                logger.error(errMsg);
+                continue;
+            }
+            const displayName = content.names?.[0]?.displayName || 'Unknown';
+
+            changes.push({
+                type: 'add',
+                resourceName: '',  // Will be assigned by Google
+                displayName,
+                filePath
+            });
+        }
+    }
+
+    return { changes, parseErrors };
 }
 
 async function confirm(message: string): Promise<boolean> {
+    // Temporarily disable raw mode for readline to work
+    const wasRawMode = process.stdin.isTTY && (process.stdin as any).isRaw;
+    if (wasRawMode) {
+        process.stdin.setRawMode(false);
+    }
+
     const readline = await import('readline');
     const rl = readline.createInterface({
         input: process.stdin,
@@ -394,55 +643,182 @@ async function confirm(message: string): Promise<boolean> {
     return new Promise(resolve => {
         rl.question(`${message} (y/N): `, answer => {
             rl.close();
+            // Restore raw mode if it was enabled
+            if (wasRawMode && process.stdin.isTTY) {
+                process.stdin.setRawMode(true);
+            }
             resolve(answer.toLowerCase() === 'y' || answer.toLowerCase() === 'yes');
         });
     });
 }
 
-async function updateContactOnGoogle(accessToken: string, person: GooglePerson): Promise<void> {
-    const updateMask = 'names,emailAddresses,phoneNumbers,addresses,organizations,biographies,urls,birthdays';
+async function updateContactOnGoogle(person: GooglePerson, retryCount = 0, tokenRefreshed = false): Promise<void> {
+    const updateMask = 'names,emailAddresses,phoneNumbers,addresses,organizations,biographies,urls,birthdays,fileAses';
 
     const response = await fetch(`${PEOPLE_API_BASE}/${person.resourceName}:updateContact?updatePersonFields=${updateMask}`, {
         method: 'PATCH',
         headers: {
-            'Authorization': `Bearer ${accessToken}`,
+            'Authorization': `Bearer ${currentAccessToken}`,
             'Content-Type': 'application/json'
         },
         body: JSON.stringify(person)
     });
 
+    // Token expired - refresh and retry once
+    if (response.status === 401 && !tokenRefreshed) {
+        await refreshAccessToken();
+        return updateContactOnGoogle(person, retryCount, true);
+    }
+
+    if (response.status === 429) {
+        if (retryCount >= 3 || escapePressed) {
+            throw new Error(`rate limit - try again later`);
+        }
+        const delay = Math.pow(2, retryCount + 1) * 10000;  // 20s, 40s, 80s
+        console.log(` ${ts()} rate limited, waiting ${delay / 1000}s (ESC to stop)...`);
+        await sleep(delay);
+        if (escapePressed) throw new Error(`stopped by user`);
+        console.log(`${ts()} retrying...`);
+        return updateContactOnGoogle(person, retryCount + 1, tokenRefreshed);
+    }
+
     if (!response.ok) {
-        throw new Error(`Update failed for ${person.resourceName}: ${response.status} ${await response.text()}`);
+        throw new Error(`${response.status} ${await response.text()}`);
     }
 }
 
-async function deleteContactOnGoogle(accessToken: string, resourceName: string): Promise<void> {
+async function deleteContactOnGoogle(resourceName: string, retryCount = 0, tokenRefreshed = false): Promise<void> {
     const response = await fetch(`${PEOPLE_API_BASE}/${resourceName}:deleteContact`, {
         method: 'DELETE',
-        headers: { 'Authorization': `Bearer ${accessToken}` }
+        headers: { 'Authorization': `Bearer ${currentAccessToken}` }
     });
 
+    // Already deleted - treat as success
+    if (response.status === 404) {
+        console.log(' (already deleted)');
+        return;
+    }
+
+    // Token expired - refresh and retry once
+    if (response.status === 401 && !tokenRefreshed) {
+        await refreshAccessToken();
+        return deleteContactOnGoogle(resourceName, retryCount, true);
+    }
+
+    if (response.status === 429) {
+        if (retryCount >= 3 || escapePressed) {
+            throw new Error(`rate limit - try again later`);
+        }
+        const delay = Math.pow(2, retryCount + 1) * 10000;  // 20s, 40s, 80s
+        console.log(` ${ts()} rate limited, waiting ${delay / 1000}s (ESC to stop)...`);
+        await sleep(delay);
+        if (escapePressed) throw new Error(`stopped by user`);
+        console.log(`${ts()} retrying...`);
+        return deleteContactOnGoogle(resourceName, retryCount + 1, tokenRefreshed);
+    }
+
     if (!response.ok) {
-        throw new Error(`Delete failed for ${resourceName}: ${response.status} ${await response.text()}`);
+        throw new Error(`${response.status} ${await response.text()}`);
     }
 }
 
-async function pushContacts(user: string, options: { yes: boolean; verbose: boolean }): Promise<void> {
+async function createContactOnGoogle(person: GooglePerson, retryCount = 0, tokenRefreshed = false): Promise<GooglePerson> {
+    // Remove resourceName and etag - Google will assign new ones
+    const { resourceName, etag, ...personData } = person as any;
+
+    // Add a GUID if not already present in userDefined
+    if (!personData.userDefined) {
+        personData.userDefined = [];
+    }
+    const hasGuid = personData.userDefined.some((u: any) => u.key === GCARDS_GUID_KEY);
+    if (!hasGuid) {
+        personData.userDefined.push({ key: GCARDS_GUID_KEY, value: generateGuid() });
+    }
+
+    const response = await fetch(`${PEOPLE_API_BASE}/people:createContact`, {
+        method: 'POST',
+        headers: {
+            'Authorization': `Bearer ${currentAccessToken}`,
+            'Content-Type': 'application/json'
+        },
+        body: JSON.stringify(personData)
+    });
+
+    // Token expired - refresh and retry once
+    if (response.status === 401 && !tokenRefreshed) {
+        await refreshAccessToken();
+        return createContactOnGoogle(person, retryCount, true);
+    }
+
+    if (response.status === 429) {
+        if (retryCount >= 3 || escapePressed) {
+            throw new Error(`rate limit - try again later`);
+        }
+        const delay = Math.pow(2, retryCount + 1) * 10000;  // 20s, 40s, 80s
+        console.log(` ${ts()} rate limited, waiting ${delay / 1000}s (ESC to stop)...`);
+        await sleep(delay);
+        if (escapePressed) throw new Error(`stopped by user`);
+        console.log(`${ts()} retrying...`);
+        return createContactOnGoogle(person, retryCount + 1, tokenRefreshed);
+    }
+
+    if (!response.ok) {
+        throw new Error(`${response.status} ${await response.text()}`);
+    }
+
+    return await response.json();
+}
+
+async function pushContacts(user: string, options: { yes: boolean; verbose: boolean; limit: number; since: string }): Promise<void> {
     const paths = getUserPaths(user);
     ensureUserDir(user);
 
+    // Create logger for problems
+    const problemsFile = path.join(paths.userDir, 'pushproblems.txt');
+    const logger = new FileLogger(problemsFile, true);
+
     console.log(`Pushing contacts for user: ${user}`);
-    const changes = findPendingChanges(paths);
+    let { changes, parseErrors } = await findPendingChanges(paths, logger, options.since || undefined);
+
+    if (parseErrors.length > 0) {
+        console.log(`\n${parseErrors.length} files had parse errors (see pushproblems.txt)`);
+    }
 
     if (changes.length === 0) {
         console.log('No pending changes to push.');
         return;
     }
 
-    const updates = changes.filter(c => c.type === 'update');
-    const deletes = changes.filter(c => c.type === 'delete');
+    const adds = changes.filter(c => c.type === 'add').sort((a, b) => a.displayName.localeCompare(b.displayName));
+    const updates = changes.filter(c => c.type === 'update').sort((a, b) => a.displayName.localeCompare(b.displayName));
+    const deletes = changes.filter(c => c.type === 'delete').sort((a, b) => a.displayName.localeCompare(b.displayName));
+
+    // Write to toPush.txt for debugging
+    const toPushLines: string[] = [];
+    if (adds.length > 0) {
+        toPushLines.push(`New contacts (${adds.length}):`);
+        for (const c of adds) toPushLines.push(`  ${c.displayName}`);
+        toPushLines.push('');
+    }
+    if (updates.length > 0) {
+        toPushLines.push(`Updates (${updates.length}):`);
+        for (const c of updates) toPushLines.push(`  ${c.displayName} (${c.resourceName})`);
+        toPushLines.push('');
+    }
+    if (deletes.length > 0) {
+        toPushLines.push(`Deletions (${deletes.length}):`);
+        for (const c of deletes) toPushLines.push(`  ${c.displayName} (${c.resourceName})`);
+    }
+    const toPushFile = path.join(paths.userDir, 'toPush.txt');
+    fs.writeFileSync(toPushFile, toPushLines.join('\n'));
 
     console.log('\nPending changes:');
+    if (adds.length > 0) {
+        console.log(`\n  New contacts (${adds.length}):`);
+        for (const c of adds) {
+            console.log(`    - ${c.displayName}`);
+        }
+    }
     if (updates.length > 0) {
         console.log(`\n  Updates (${updates.length}):`);
         for (const c of updates) {
@@ -455,6 +831,10 @@ async function pushContacts(user: string, options: { yes: boolean; verbose: bool
             console.log(`    - ${c.displayName} (${c.resourceName})`);
         }
     }
+    console.log(`\n(List written to ${toPushFile})`);
+
+    // Reorder changes to process in sorted order
+    changes = [...adds, ...updates, ...deletes];
 
     if (!options.yes) {
         const confirmed = await confirm(`\nPush ${changes.length} change(s) to Google?`);
@@ -465,110 +845,265 @@ async function pushContacts(user: string, options: { yes: boolean; verbose: bool
     }
 
     console.log('\nGetting access token (write access)...');
-    const accessToken = await getAccessToken(user, true);
+    currentUser = user;
+    currentAccessToken = await getAccessToken(user, true);
 
     const index = loadIndex(paths);
+    const deletedIndex = loadDeleted(paths);
+    const deleteQueue = loadDeleteQueue(paths);
+    const completedDeletions = new Set<string>();  // Track successful deletions
+    // Apply limit if specified
+    if (options.limit > 0 && changes.length > options.limit) {
+        console.log(`[DEBUG] Limiting to first ${options.limit} of ${changes.length} changes\n`);
+        changes = changes.slice(0, options.limit);
+    }
+
     let successCount = 0;
     let errorCount = 0;
+    let skippedPhoto = 0;
+    let skippedStarred = 0;
+    let processed = 0;
+    const total = changes.length;
+    const problems: string[] = [];
+    const notDeleted: { resourceName: string; displayName: string; filePath: string; reason: string }[] = [];
 
     for (const change of changes) {
+        if (escapePressed) {
+            console.log('\n\nStopped by user. Progress saved.');
+            break;
+        }
+
+        processed++;
         try {
-            if (change.type === 'update') {
+            if (change.type === 'add') {
                 const content = JSON.parse(fs.readFileSync(change.filePath, 'utf-8')) as GooglePerson;
-                process.stdout.write(`Updating ${change.displayName}...`);
-                await updateContactOnGoogle(accessToken, content);
-                console.log(' done');
+                process.stdout.write(`${ts()} [${String(processed).padStart(5)}/${total}] Creating ${change.displayName}...`);
+                const created = await createContactOnGoogle(content);
+
+                // Extract GUIDs and flags from created contact
+                const guids = extractGuids(created);
+                const hasPhoto = hasNonDefaultPhoto(created);
+                const starred = isStarred(created);
+
+                // Add to index
+                index.contacts[created.resourceName] = {
+                    resourceName: created.resourceName,
+                    displayName: change.displayName,
+                    updatedAt: new Date().toISOString(),
+                    ...(hasPhoto && { hasPhoto }),
+                    ...(starred && { starred }),
+                    ...(guids.length > 0 && { guids })
+                };
+
+                // Save new contact to contacts/ folder
+                const newFilePath = path.join(paths.contactsDir, `${created.resourceName.replace(/\//g, '_')}.json`);
+                if (!fs.existsSync(paths.contactsDir)) {
+                    fs.mkdirSync(paths.contactsDir, { recursive: true });
+                }
+                fs.writeFileSync(newFilePath, JSON.stringify(created, null, 2));
+
+                // Remove from _add/ folder
+                fs.unlinkSync(change.filePath);
+
+                console.log(` done (${created.resourceName})`);
                 successCount++;
-            } else {
-                process.stdout.write(`Deleting ${change.displayName}...`);
-                await deleteContactOnGoogle(accessToken, change.resourceName);
+            } else if (change.type === 'update') {
+                const content = JSON.parse(fs.readFileSync(change.filePath, 'utf-8')) as GooglePerson;
+                process.stdout.write(`${ts()} [${String(processed).padStart(5)}/${total}] Updating ${change.displayName}...`);
+                await updateContactOnGoogle(content);
 
-                // Update index
+                // Update index timestamp so file no longer appears modified
                 if (index.contacts[change.resourceName]) {
-                    index.contacts[change.resourceName].deleted = true;
-                    index.contacts[change.resourceName].deletedAt = new Date().toISOString();
-                }
-
-                // Remove from deleted/ folder if it was there
-                if (change.filePath.startsWith(paths.deletedDir)) {
-                    fs.unlinkSync(change.filePath);
+                    index.contacts[change.resourceName].updatedAt = new Date().toISOString();
                 }
 
                 console.log(' done');
                 successCount++;
+            } else if (change.type === 'delete') {
+                const contactFile = path.join(paths.contactsDir, `${change.resourceName.replace('people/', '')}.json`);
+                const fileToRead = fs.existsSync(contactFile) ? contactFile : change.filePath;
+
+                // Check if contact has a real photo or is starred - skip deletion if so
+                let contactHasPhoto = false;
+                let contactIsStarred = false;
+                if (fs.existsSync(fileToRead)) {
+                    try {
+                        const contact = JSON.parse(fs.readFileSync(fileToRead, 'utf-8')) as GooglePerson;
+                        contactHasPhoto = hasNonDefaultPhoto(contact);
+                        contactIsStarred = isStarred(contact);
+                    } catch { /* ignore read errors */ }
+                }
+
+                // Determine skip reason (photo takes precedence over starred)
+                // Use * prefix to indicate "skipped, not deleted"
+                // 'force' overrides photo/starred protection
+                const isForced = change._delete === 'force' || index.contacts[change.resourceName]?._delete === 'force';
+                const skipReason = isForced ? null : (contactHasPhoto ? '*photo' : (contactIsStarred ? '*starred' : null));
+
+                if (skipReason) {
+                    // Skip Google deletion - mark why in index.json
+                    notDeleted.push({
+                        resourceName: change.resourceName,
+                        displayName: change.displayName,
+                        filePath: fileToRead,
+                        reason: skipReason
+                    });
+                    if (skipReason === '*photo') skippedPhoto++;
+                    else if (skipReason === '*starred') skippedStarred++;
+                    console.log(`${ts()} [${String(processed).padStart(5)}/${total}] SKIPPED ${change.displayName} (${skipReason})`);
+
+                    // Update _delete to show skip reason
+                    if (index.contacts[change.resourceName]) {
+                        index.contacts[change.resourceName]._delete = skipReason;
+                    }
+
+                    // Remove from _delete/ folder if it was there (keep in contacts/)
+                    if (change.filePath !== fileToRead && change.filePath.startsWith(paths.toDeleteDir) && fs.existsSync(change.filePath)) {
+                        fs.unlinkSync(change.filePath);
+                    }
+
+                    completedDeletions.add(change.resourceName);  // Remove from queue even if skipped
+                } else {
+                    process.stdout.write(`${ts()} [${String(processed).padStart(5)}/${total}] Deleting ${change.displayName}...`);
+                    await deleteContactOnGoogle(change.resourceName);
+
+                    // Move to deleted.json (preserve _delete reason)
+                    const entry = index.contacts[change.resourceName];
+                    if (entry) {
+                        const deletedEntry = { ...entry, deletedAt: new Date().toISOString() };
+                        // Keep _delete reason in deleted entry (it's the reason for deletion)
+                        deletedIndex.deleted[change.resourceName] = deletedEntry;
+                        delete index.contacts[change.resourceName];
+                    }
+
+                    // Move contact file to deleted/ folder (backup with timestamp)
+                    if (!fs.existsSync(paths.deletedDir)) {
+                        fs.mkdirSync(paths.deletedDir, { recursive: true });
+                    }
+                    if (fs.existsSync(contactFile)) {
+                        // Add deletion metadata to the JSON file
+                        const contactData = JSON.parse(fs.readFileSync(contactFile, 'utf-8'));
+                        contactData._deletedAt = new Date().toISOString();
+                        contactData._delete = change._delete || entry?._delete || 'unknown';
+                        const destPath = path.join(paths.deletedDir, path.basename(contactFile));
+                        fs.writeFileSync(destPath, JSON.stringify(contactData, null, 2));
+                        fs.unlinkSync(contactFile);
+                    }
+
+                    // Remove from _delete/ folder if it was there
+                    if (change.filePath.startsWith(paths.toDeleteDir) && fs.existsSync(change.filePath)) {
+                        fs.unlinkSync(change.filePath);
+                    }
+
+                    completedDeletions.add(change.resourceName);
+                    console.log(' done');
+                    successCount++;
+                }
             }
 
-            await sleep(200);  // Be nice to the API
+            await sleep(700);  // 90 writes/min limit = ~700ms between ops
         } catch (error: any) {
+            const fileName = path.basename(change.filePath);
+            const errorMsg = `${fileName}: ${error.message}`;
             console.log(` ERROR: ${error.message}`);
+            problems.push(errorMsg);
             errorCount++;
-            debugger;
         }
     }
 
-    // Save status with new timestamp
-    saveStatus(paths, { lastPush: new Date().toISOString() });
-    saveIndex(paths, index);
-
-    console.log(`\nPush complete: ${successCount} succeeded, ${errorCount} failed`);
-}
+    // Write problems to file if any
+    if (problems.length > 0) {
+        const problemsFile = path.join(paths.userDir, 'pushproblems.txt');
+        fs.writeFileSync(problemsFile, problems.join('\n'));
+        console.log(`\nErrors written to: ${problemsFile}`);
+    }
 
-function resolveUser(cliUser: string): string {
-    // 1. Use CLI-specified user if provided (and not 'default')
-    if (cliUser && cliUser !== 'default') {
-        const normalized = normalizeUser(cliUser);
-        const config = loadConfig();
-        config.lastUser = normalized;
-        saveConfig(config);
-        return normalized;
+    // Write notdeleted file if any contacts were skipped due to photos
+    if (notDeleted.length > 0) {
+        const notDeletedFile = path.join(paths.userDir, 'notdeleted.json');
+        fs.writeFileSync(notDeletedFile, JSON.stringify(notDeleted, null, 2));
+        console.log(`\n${notDeleted.length} contacts with photos not deleted from Google (see notdeleted.json)`);
     }
 
-    // 2. Use lastUser from config (already normalized)
-    const config = loadConfig();
-    if (config.lastUser) {
-        return config.lastUser;
+    // Remove completed deletions from queue
+    if (completedDeletions.size > 0) {
+        deleteQueue.entries = deleteQueue.entries.filter(e => !completedDeletions.has(e.resourceName));
+        saveDeleteQueue(paths, deleteQueue);
     }
 
-    // 3. No user - require --user on first use
-    console.error('No user specified. Use --user <name> on first run (e.g., your Gmail username).');
-    console.error('This will be remembered for future runs.');
-    process.exit(1);
+    // Save status with new timestamp
+    saveStatus(paths, { lastPush: new Date().toISOString() });
+    saveIndex(paths, index);
+    saveDeleted(paths, deletedIndex);
+
+    const skipped = skippedPhoto + skippedStarred;
+    const skippedDetails = skipped > 0 ? ` (${skippedPhoto} photo, ${skippedStarred} starred)` : '';
+    console.log(`\nPush complete: ${successCount} succeeded, ${errorCount} failed, ${skipped} skipped${skippedDetails}`);
 }
 
 async function main(): Promise<void> {
     const args = process.argv.slice(2);
     const options = parseArgs(args);
 
-    if (options.help || !options.command) {
+    if (options.help) {
         showHelp();
         return;
     }
 
+    if (!options.command) {
+        showUsage();
+        return;
+    }
+
+    setupEscapeHandler();
+
     try {
-        const user = resolveUser(options.user);
-
-        switch (options.command) {
-            case 'sync':
-                await syncContacts(user, { full: options.full, verbose: options.verbose });
-                break;
-            case 'push':
-                await pushContacts(user, { yes: options.yes, verbose: options.verbose });
-                break;
-            default:
-                console.error(`Unknown command: ${options.command}`);
-                showHelp();
+        // Determine users to process
+        let users: string[];
+        if (options.all) {
+            users = getAllUsers();
+            if (users.length === 0) {
+                console.error('No users found. Use --user <name> to create one first.');
                 process.exit(1);
+            }
+            console.log(`Processing all users: ${users.join(', ')}\n`);
+        } else {
+            users = [resolveUser(options.user, true)];
         }
 
-        // Update lastUser in config
-        const config = loadConfig();
-        config.lastUser = user;
-        saveConfig(config);
+        for (const user of users) {
+            if (escapePressed) break;
+
+            if (users.length > 1) {
+                console.log(`\n${'='.repeat(50)}\nUser: ${user}\n${'='.repeat(50)}`);
+            }
+
+            switch (options.command) {
+                case 'sync':
+                    await syncContacts(user, { full: options.full, verbose: options.verbose });
+                    break;
+                case 'push':
+                    await pushContacts(user, { yes: options.yes, verbose: options.verbose, limit: options.limit, since: options.since });
+                    break;
+                default:
+                    console.error(`Unknown command: ${options.command}`);
+                    showHelp();
+                    process.exit(1);
+            }
+        }
+
+        // Update lastUser in config (use last processed user)
+        if (!options.all && users.length === 1) {
+            const config = loadConfig();
+            config.lastUser = users[0];
+            saveConfig(config);
+        }
 
     } catch (error: any) {
         console.error(`Error: ${error.message}`);
-        debugger;
         process.exit(1);
+    } finally {
+        cleanupEscapeHandler();
     }
 }