@indigoai-us/hq-cloud 5.1.0

package/src/index.ts

@@ -0,0 +1,182 @@
+/**
+ * @indigoai-us/hq-cloud — public API
+ * Used by @indigoai-us/hq-cli to manage cloud sync
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import { authenticate, hasCredentials, readCredentials } from "./auth.js";
+import {
+  startDaemon as _startDaemon,
+  stopDaemon as _stopDaemon,
+  isDaemonRunning,
+} from "./daemon.js";
+import { readJournal, writeJournal, hashFile, updateEntry } from "./journal.js";
+import { uploadFile, downloadFile, listRemoteFiles } from "./s3.js";
+import { createIgnoreFilter, isWithinSizeLimit } from "./ignore.js";
+import type { SyncStatus, PushResult, PullResult } from "./types.js";
+
+export type { SyncStatus, PushResult, PullResult } from "./types.js";
+
+// Cognito identity helpers — used by `hq auth refresh` and any consumer
+// that needs a valid HQ access token (deploy skill, onboarding, etc.).
+export {
+  browserLogin,
+  refreshTokens,
+  getValidAccessToken,
+  loadCachedTokens,
+  saveCachedTokens,
+  clearCachedTokens,
+  isExpiring,
+  CognitoAuthError,
+} from "./cognito-auth.js";
+export type { CognitoAuthConfig, CognitoTokens } from "./cognito-auth.js";
+
+/**
+ * Initialize cloud sync — authenticate and provision bucket
+ */
+export async function initSync(hqRoot: string): Promise<void> {
+  if (hasCredentials()) {
+    console.log("  Already authenticated. Use 'hq sync start' to begin syncing.");
+    return;
+  }
+
+  console.log("  Setting up IndigoAI cloud sync...");
+  const creds = await authenticate();
+  console.log(`  ✓ Authenticated as ${creds.userId}`);
+  console.log(`  ✓ Bucket: ${creds.bucket}`);
+  console.log(`  ✓ Region: ${creds.region}`);
+  console.log();
+  console.log("  Run 'hq sync start' to begin syncing.");
+}
+
+/**
+ * Start the background sync daemon
+ */
+export async function startDaemon(hqRoot: string): Promise<void> {
+  if (!hasCredentials()) {
+    throw new Error("Not authenticated. Run 'hq sync init' first.");
+  }
+  _startDaemon(hqRoot);
+}
+
+/**
+ * Stop the background sync daemon
+ */
+export async function stopDaemon(hqRoot: string): Promise<void> {
+  _stopDaemon(hqRoot);
+}
+
+/**
+ * Get current sync status
+ */
+export async function getStatus(hqRoot: string): Promise<SyncStatus> {
+  const journal = readJournal(hqRoot);
+  const creds = readCredentials();
+  const running = isDaemonRunning(hqRoot);
+  const errors: string[] = [];
+
+  if (!creds) {
+    errors.push("Not authenticated — run 'hq sync init'");
+  }
+
+  return {
+    running,
+    lastSync: journal.lastSync || null,
+    fileCount: Object.keys(journal.files).length,
+    bucket: creds?.bucket || null,
+    errors,
+  };
+}
+
+/**
+ * Force push all local files to S3
+ */
+export async function pushAll(hqRoot: string): Promise<PushResult> {
+  const shouldSync = createIgnoreFilter(hqRoot);
+  const journal = readJournal(hqRoot);
+  let filesUploaded = 0;
+  let bytesUploaded = 0;
+
+  const files = walkDir(hqRoot, hqRoot, shouldSync);
+
+  for (const { absolutePath, relativePath } of files) {
+    if (!isWithinSizeLimit(absolutePath)) continue;
+
+    try {
+      const hash = hashFile(absolutePath);
+      const stat = fs.statSync(absolutePath);
+
+      await uploadFile(absolutePath, relativePath);
+      updateEntry(journal, relativePath, hash, stat.size, "up");
+      filesUploaded++;
+      bytesUploaded += stat.size;
+    } catch (err) {
+      console.error(
+        `  Failed: ${relativePath} — ${err instanceof Error ? err.message : err}`
+      );
+    }
+  }
+
+  writeJournal(hqRoot, journal);
+  return { filesUploaded, bytesUploaded };
+}
+
+/**
+ * Force pull all remote files to local
+ */
+export async function pullAll(hqRoot: string): Promise<PullResult> {
+  const journal = readJournal(hqRoot);
+  let filesDownloaded = 0;
+  let bytesDownloaded = 0;
+
+  const remoteFiles = await listRemoteFiles();
+
+  for (const file of remoteFiles) {
+    try {
+      const localPath = path.join(hqRoot, file.relativePath);
+      await downloadFile(file.relativePath, localPath);
+
+      const hash = hashFile(localPath);
+      updateEntry(journal, file.relativePath, hash, file.size, "down");
+      filesDownloaded++;
+      bytesDownloaded += file.size;
+    } catch (err) {
+      console.error(
+        `  Failed: ${file.relativePath} — ${err instanceof Error ? err.message : err}`
+      );
+    }
+  }
+
+  writeJournal(hqRoot, journal);
+  return { filesDownloaded, bytesDownloaded };
+}
+
+// Helper: recursively walk a directory
+function walkDir(
+  dir: string,
+  root: string,
+  filter: (p: string) => boolean
+): { absolutePath: string; relativePath: string }[] {
+  const results: { absolutePath: string; relativePath: string }[] = [];
+
+  if (!fs.existsSync(dir)) return results;
+
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const absolutePath = path.join(dir, entry.name);
+
+    if (!filter(absolutePath)) continue;
+
+    if (entry.isDirectory()) {
+      results.push(...walkDir(absolutePath, root, filter));
+    } else if (entry.isFile()) {
+      results.push({
+        absolutePath,
+        relativePath: path.relative(root, absolutePath),
+      });
+    }
+  }
+
+  return results;
+}

package/src/journal.ts

@@ -0,0 +1,63 @@
+/**
+ * Sync journal — tracks file state for conflict detection
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import * as crypto from "crypto";
+import type { SyncJournal, JournalEntry } from "./types.js";
+
+const JOURNAL_FILE = ".hq-sync-journal.json";
+
+export function getJournalPath(hqRoot: string): string {
+  return path.join(hqRoot, JOURNAL_FILE);
+}
+
+export function readJournal(hqRoot: string): SyncJournal {
+  const journalPath = getJournalPath(hqRoot);
+  if (fs.existsSync(journalPath)) {
+    const content = fs.readFileSync(journalPath, "utf-8");
+    return JSON.parse(content) as SyncJournal;
+  }
+  return { version: "1", lastSync: "", files: {} };
+}
+
+export function writeJournal(hqRoot: string, journal: SyncJournal): void {
+  const journalPath = getJournalPath(hqRoot);
+  fs.writeFileSync(journalPath, JSON.stringify(journal, null, 2));
+}
+
+export function hashFile(filePath: string): string {
+  const content = fs.readFileSync(filePath);
+  return crypto.createHash("sha256").update(content).digest("hex");
+}
+
+export function updateEntry(
+  journal: SyncJournal,
+  relativePath: string,
+  hash: string,
+  size: number,
+  direction: "up" | "down"
+): void {
+  journal.files[relativePath] = {
+    hash,
+    size,
+    syncedAt: new Date().toISOString(),
+    direction,
+  };
+  journal.lastSync = new Date().toISOString();
+}
+
+export function getEntry(
+  journal: SyncJournal,
+  relativePath: string
+): JournalEntry | undefined {
+  return journal.files[relativePath];
+}
+
+export function removeEntry(
+  journal: SyncJournal,
+  relativePath: string
+): void {
+  delete journal.files[relativePath];
+}

package/src/s3.ts

@@ -0,0 +1,178 @@
+/**
+ * S3 operations — upload, download, list, delete
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import {
+  S3Client,
+  PutObjectCommand,
+  GetObjectCommand,
+  ListObjectsV2Command,
+  DeleteObjectCommand,
+} from "@aws-sdk/client-s3";
+import type { Credentials, SyncConfig } from "./types.js";
+import { readCredentials, refreshAwsCredentials } from "./auth.js";
+
+let s3Client: S3Client | null = null;
+
+function getConfig(creds: Credentials): SyncConfig {
+  const prefix = creds.teamId
+    ? `teams/${creds.teamId}/users/${creds.userId}/hq/`
+    : `users/${creds.userId}/hq/`;
+  return {
+    bucket: creds.bucket,
+    region: creds.region,
+    userId: creds.userId,
+    prefix,
+  };
+}
+
+async function getClient(): Promise<{ client: S3Client; config: SyncConfig }> {
+  let creds = readCredentials();
+  if (!creds) {
+    throw new Error("Not authenticated. Run 'hq sync init' first.");
+  }
+
+  // Refresh if expired or missing access key
+  if (!creds.accessKeyId || (creds.expiration && new Date(creds.expiration) < new Date())) {
+    creds = await refreshAwsCredentials(creds);
+  }
+
+  if (!s3Client) {
+    s3Client = new S3Client({
+      region: creds.region,
+      credentials: {
+        accessKeyId: creds.accessKeyId,
+        secretAccessKey: creds.secretAccessKey,
+        sessionToken: creds.sessionToken,
+      },
+    });
+  }
+
+  return { client: s3Client, config: getConfig(creds) };
+}
+
+export async function uploadFile(
+  localPath: string,
+  relativePath: string
+): Promise<void> {
+  const { client, config } = await getClient();
+  const key = `${config.prefix}${relativePath}`;
+  const body = fs.readFileSync(localPath);
+
+  await client.send(
+    new PutObjectCommand({
+      Bucket: config.bucket,
+      Key: key,
+      Body: body,
+      ContentType: getMimeType(relativePath),
+    })
+  );
+}
+
+export async function downloadFile(
+  relativePath: string,
+  localPath: string
+): Promise<void> {
+  const { client, config } = await getClient();
+  const key = `${config.prefix}${relativePath}`;
+
+  const response = await client.send(
+    new GetObjectCommand({
+      Bucket: config.bucket,
+      Key: key,
+    })
+  );
+
+  if (!response.Body) {
+    throw new Error(`Empty response for ${key}`);
+  }
+
+  const dir = path.dirname(localPath);
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+
+  const chunks: Buffer[] = [];
+  const stream = response.Body as AsyncIterable<Uint8Array>;
+  for await (const chunk of stream) {
+    chunks.push(Buffer.from(chunk));
+  }
+  fs.writeFileSync(localPath, Buffer.concat(chunks));
+}
+
+export interface RemoteFile {
+  key: string;
+  relativePath: string;
+  size: number;
+  lastModified: Date;
+  etag: string;
+}
+
+export async function listRemoteFiles(): Promise<RemoteFile[]> {
+  const { client, config } = await getClient();
+  const files: RemoteFile[] = [];
+  let continuationToken: string | undefined;
+
+  do {
+    const response = await client.send(
+      new ListObjectsV2Command({
+        Bucket: config.bucket,
+        Prefix: config.prefix,
+        ContinuationToken: continuationToken,
+      })
+    );
+
+    for (const obj of response.Contents || []) {
+      if (!obj.Key || !obj.Size) continue;
+      const relativePath = obj.Key.replace(config.prefix, "");
+      if (!relativePath) continue;
+
+      files.push({
+        key: obj.Key,
+        relativePath,
+        size: obj.Size,
+        lastModified: obj.LastModified || new Date(),
+        etag: obj.ETag || "",
+      });
+    }
+
+    continuationToken = response.NextContinuationToken;
+  } while (continuationToken);
+
+  return files;
+}
+
+export async function deleteRemoteFile(relativePath: string): Promise<void> {
+  const { client, config } = await getClient();
+  const key = `${config.prefix}${relativePath}`;
+
+  await client.send(
+    new DeleteObjectCommand({
+      Bucket: config.bucket,
+      Key: key,
+    })
+  );
+}
+
+function getMimeType(filePath: string): string {
+  const ext = path.extname(filePath).toLowerCase();
+  const mimeTypes: Record<string, string> = {
+    ".md": "text/markdown",
+    ".json": "application/json",
+    ".yaml": "text/yaml",
+    ".yml": "text/yaml",
+    ".ts": "text/typescript",
+    ".js": "text/javascript",
+    ".txt": "text/plain",
+    ".html": "text/html",
+    ".css": "text/css",
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".svg": "image/svg+xml",
+    ".pdf": "application/pdf",
+  };
+  return mimeTypes[ext] || "application/octet-stream";
+}

package/src/types.ts

@@ -0,0 +1,59 @@
+/**
+ * HQ Cloud Sync Types
+ */
+
+export interface SyncConfig {
+  bucket: string;
+  region: string;
+  userId: string;
+  prefix: string; // e.g. "hq/"
+}
+
+export interface Credentials {
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken?: string;
+  expiration?: string;
+  refreshToken: string;
+  userId: string;
+  bucket: string;
+  region: string;
+  teamId?: string;
+}
+
+export interface JournalEntry {
+  hash: string;
+  size: number;
+  syncedAt: string;
+  direction: "up" | "down";
+}
+
+export interface SyncJournal {
+  version: "1";
+  lastSync: string;
+  files: Record<string, JournalEntry>;
+}
+
+export interface SyncStatus {
+  running: boolean;
+  lastSync: string | null;
+  fileCount: number;
+  bucket: string | null;
+  errors: string[];
+}
+
+export interface PushResult {
+  filesUploaded: number;
+  bytesUploaded: number;
+}
+
+export interface PullResult {
+  filesDownloaded: number;
+  bytesDownloaded: number;
+}
+
+export interface DaemonState {
+  pid: number;
+  startedAt: string;
+  hqRoot: string;
+}

package/src/watcher.ts

@@ -0,0 +1,130 @@
+/**
+ * File watcher — monitors HQ directory for changes
+ * Uses chokidar with debounced batching
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import { watch } from "chokidar";
+import type { FSWatcher } from "chokidar";
+import { createIgnoreFilter, isWithinSizeLimit } from "./ignore.js";
+import { readJournal, writeJournal, hashFile, updateEntry } from "./journal.js";
+import { uploadFile, deleteRemoteFile } from "./s3.js";
+
+const DEBOUNCE_MS = 2000;
+
+interface PendingChange {
+  type: "add" | "change" | "unlink";
+  absolutePath: string;
+  relativePath: string;
+}
+
+export class SyncWatcher {
+  private watcher: FSWatcher | null = null;
+  private hqRoot: string;
+  private shouldSync: (filePath: string) => boolean;
+  private pendingChanges = new Map<string, PendingChange>();
+  private debounceTimer: ReturnType<typeof setTimeout> | null = null;
+  private processing = false;
+
+  constructor(hqRoot: string) {
+    this.hqRoot = hqRoot;
+    this.shouldSync = createIgnoreFilter(hqRoot);
+  }
+
+  start(): void {
+    if (this.watcher) return;
+
+    this.watcher = watch(this.hqRoot, {
+      ignored: (filePath: string) => !this.shouldSync(filePath),
+      persistent: true,
+      ignoreInitial: true,
+      awaitWriteFinish: {
+        stabilityThreshold: 500,
+        pollInterval: 100,
+      },
+    });
+
+    this.watcher
+      .on("add", (p) => this.queueChange("add", p))
+      .on("change", (p) => this.queueChange("change", p))
+      .on("unlink", (p) => this.queueChange("unlink", p))
+      .on("error", (err) => console.error("Watcher error:", err));
+  }
+
+  stop(): void {
+    if (this.watcher) {
+      this.watcher.close();
+      this.watcher = null;
+    }
+    if (this.debounceTimer) {
+      clearTimeout(this.debounceTimer);
+      this.debounceTimer = null;
+    }
+  }
+
+  private queueChange(type: "add" | "change" | "unlink", absolutePath: string): void {
+    const relativePath = path.relative(this.hqRoot, absolutePath);
+
+    // Skip files that exceed size limit
+    if (type !== "unlink" && !isWithinSizeLimit(absolutePath)) {
+      return;
+    }
+
+    this.pendingChanges.set(relativePath, {
+      type,
+      absolutePath,
+      relativePath,
+    });
+
+    // Debounce: wait for DEBOUNCE_MS of quiet before processing
+    if (this.debounceTimer) {
+      clearTimeout(this.debounceTimer);
+    }
+    this.debounceTimer = setTimeout(() => this.flush(), DEBOUNCE_MS);
+  }
+
+  private async flush(): Promise<void> {
+    if (this.processing || this.pendingChanges.size === 0) return;
+    this.processing = true;
+
+    const batch = new Map(this.pendingChanges);
+    this.pendingChanges.clear();
+
+    const journal = readJournal(this.hqRoot);
+
+    for (const [relativePath, change] of batch) {
+      try {
+        if (change.type === "unlink") {
+          await deleteRemoteFile(relativePath);
+          delete journal.files[relativePath];
+        } else {
+          const hash = hashFile(change.absolutePath);
+          const stat = fs.statSync(change.absolutePath);
+
+          // Skip if unchanged from last sync
+          const existing = journal.files[relativePath];
+          if (existing && existing.hash === hash) continue;
+
+          await uploadFile(change.absolutePath, relativePath);
+          updateEntry(journal, relativePath, hash, stat.size, "up");
+        }
+      } catch (err) {
+        console.error(
+          `Sync error [${relativePath}]:`,
+          err instanceof Error ? err.message : err
+        );
+        // Re-queue failed changes
+        this.pendingChanges.set(relativePath, change);
+      }
+    }
+
+    writeJournal(this.hqRoot, journal);
+    this.processing = false;
+
+    // Process any changes that came in while we were flushing
+    if (this.pendingChanges.size > 0) {
+      this.debounceTimer = setTimeout(() => this.flush(), DEBOUNCE_MS);
+    }
+  }
+}

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": "src"
+  },
+  "include": ["src"]
+}