@hasna/files 0.2.17 → 0.2.19

package/dist/cli/index.js

@@ -131979,7 +131979,7 @@ var init_sync = __esm(() => {
 
 // node_modules/readdirp/esm/index.js
 import { stat, lstat, readdir, realpath } from "fs/promises";
-import { Readable as Readable4 } from "stream";
+import { Readable as Readable5 } from "stream";
 import { resolve as presolve, relative as prelative, join as pjoin, sep as psep } from "path";
 function readdirp(root2, options = {}) {
   let type = options.entryType || options.type;
@@ -132048,7 +132048,7 @@ var init_esm2 = __esm(() => {
     EntryTypes.FILE_TYPE
   ]);
   wantBigintFsStats = process.platform === "win32";
-  ReaddirpStream = class ReaddirpStream extends Readable4 {
+  ReaddirpStream = class ReaddirpStream extends Readable5 {
     constructor(options = {}) {
       super({
         objectMode: true,
@@ -134007,8 +134007,9 @@ async function walkDir(rootPath, dirPath, source, machine_id, seenPaths, stats,
 init_mime_types();
 init_machines();
 init_files();
-import { mkdirSync as mkdirSync8, writeFileSync as writeFileSync7 } from "fs";
-import { basename as basename5, dirname as dirname5, extname as extname5, join as join8, posix } from "path";
+import { createWriteStream as createWriteStream2, mkdirSync as mkdirSync10, writeFileSync as writeFileSync9 } from "fs";
+import { basename as basename5, dirname as dirname5, extname as extname5, join as join11, posix } from "path";
+import { pipeline as pipeline2 } from "stream/promises";
 
 // src/db/google-drive.ts
 init_database();
@@ -143584,6 +143585,12 @@ async function uploadBufferToS3(source, body, s3Key, contentType = "application/
   return s3Key;
 }
 
+// src/lib/google-drive-client.ts
+import { existsSync as existsSync9, mkdirSync as mkdirSync8, readFileSync as readFileSync8, writeFileSync as writeFileSync7 } from "fs";
+import { homedir as homedir7 } from "os";
+import { join as join8 } from "path";
+import { Readable as Readable4 } from "stream";
+
 // node_modules/@hasna/connectors/dist/index.js
 import { Buffer as Buffer22 } from "buffer";
 import { existsSync as existsSync5 } from "fs";
@@ -167379,6 +167386,10 @@ function runLegacyConnectorCommand(name, args, timeoutMs = 30000) {
 
 // src/lib/google-drive-client.ts
 var GOOGLE_FOLDER_MIME = "application/vnd.google-apps.folder";
+var DRIVE_API_BASE2 = "https://www.googleapis.com/drive/v3";
+var TOKEN_URL3 = "https://oauth2.googleapis.com/token";
+var REFRESH_BUFFER_MS3 = 5 * 60 * 1000;
+var RANGE_DOWNLOAD_CHUNK_BYTES = 64 * 1024 * 1024;
 var DEFAULT_EXPORT_FORMATS2 = {
   document: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
   spreadsheet: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
@@ -167450,6 +167461,38 @@ class ConnectorSdkGoogleDriveClient {
       mimeType: response.mimeType ?? file.mimeType ?? "application/octet-stream"
     };
   }
+  async downloadFileStream(file, exportFormats = {}) {
+    if (file.mimeType.startsWith("application/vnd.google-apps.")) {
+      const downloaded = await this.downloadFile(file, exportFormats);
+      return {
+        body: Readable4.from(Buffer.from(downloaded.data)),
+        filename: downloaded.filename,
+        mimeType: downloaded.mimeType,
+        size: downloaded.data.byteLength
+      };
+    }
+    const size = file.size ? Number(file.size) : undefined;
+    if (size && size > RANGE_DOWNLOAD_CHUNK_BYTES) {
+      return {
+        body: Readable4.from(downloadGoogleDriveRangeChunks(this.profile, file.id, size)),
+        filename: file.name,
+        mimeType: file.mimeType || "application/octet-stream",
+        size
+      };
+    }
+    const response = await requestGoogleDrive(this.profile, `/files/${encodeURIComponent(file.id)}`, {
+      alt: "media",
+      supportsAllDrives: true
+    });
+    if (!response.body)
+      throw new Error(`Google Drive download for "${file.name}" returned no stream`);
+    return {
+      body: Readable4.fromWeb(response.body),
+      filename: file.name,
+      mimeType: response.headers.get("content-type")?.split(";")[0] || file.mimeType || "application/octet-stream",
+      size: size ?? (Number(response.headers.get("content-length") ?? 0) || undefined)
+    };
+  }
 }
 async function runGoogleDriveOperation(operation, profile, input) {
   const result = await runConnectorOperation({
@@ -167477,9 +167520,145 @@ function getExportMimeType(googleMimeType, exportFormats) {
     return exportFormats.drawing ?? DEFAULT_EXPORT_FORMATS2.drawing;
   throw new Error(`Cannot export Google Workspace file type: ${googleMimeType}`);
 }
+async function requestGoogleDrive(profile, path, params, headers = {}) {
+  const token = await getValidAccessToken3(profile);
+  const url = new URL(`${DRIVE_API_BASE2}${path}`);
+  for (const [key, value] of Object.entries(params)) {
+    if (value !== undefined && value !== null && value !== "")
+      url.searchParams.set(key, String(value));
+  }
+  const response = await fetch(url, {
+    headers: {
+      Authorization: `Bearer ${token}`,
+      Accept: "application/octet-stream",
+      ...headers
+    }
+  });
+  if (!response.ok) {
+    const body = await response.text().catch(() => "");
+    throw new Error(`Google Drive request failed (${response.status}): ${extractGoogleError2(body) || response.statusText}`);
+  }
+  return response;
+}
+async function* downloadGoogleDriveRangeChunks(profile, fileId, size) {
+  for (let start = 0;start < size; start += RANGE_DOWNLOAD_CHUNK_BYTES) {
+    const end = Math.min(start + RANGE_DOWNLOAD_CHUNK_BYTES - 1, size - 1);
+    const response = await requestGoogleDrive(profile, `/files/${encodeURIComponent(fileId)}`, { alt: "media", supportsAllDrives: true }, { Range: `bytes=${start}-${end}` });
+    const data = Buffer.from(await response.arrayBuffer());
+    yield data;
+  }
+}
+async function getValidAccessToken3(profile) {
+  if (process.env.GOOGLE_ACCESS_TOKEN)
+    return process.env.GOOGLE_ACCESS_TOKEN;
+  const loaded = loadTokens4(profile);
+  const tokens = loaded?.tokens;
+  if (!tokens?.accessToken && !tokens?.refreshToken) {
+    throw new Error(`Google Drive profile "${profile}" is not authenticated. Run: connectors auth googledrive`);
+  }
+  if (tokens.accessToken && (!tokens.expiresAt || Date.now() < tokens.expiresAt - REFRESH_BUFFER_MS3))
+    return tokens.accessToken;
+  if (!tokens.refreshToken)
+    return tokens.accessToken ?? "";
+  return (await refreshAccessToken3(profile, tokens, loaded?.baseDir)).accessToken ?? "";
+}
+async function refreshAccessToken3(profile, currentTokens, preferredBaseDir) {
+  const credentials = loadCredentials3(profile);
+  if (!credentials.clientId || !credentials.clientSecret)
+    throw new Error("Google Drive OAuth credentials are not configured. Run: connectors auth googledrive");
+  if (!currentTokens.refreshToken)
+    throw new Error(`Google Drive profile "${profile}" has no refresh token. Run: connectors auth googledrive`);
+  const response = await fetch(TOKEN_URL3, {
+    method: "POST",
+    headers: { "Content-Type": "application/x-www-form-urlencoded" },
+    body: new URLSearchParams({
+      client_id: credentials.clientId,
+      client_secret: credentials.clientSecret,
+      refresh_token: currentTokens.refreshToken,
+      grant_type: "refresh_token"
+    })
+  });
+  const data = await response.json().catch(() => ({}));
+  if (!response.ok || !data.access_token)
+    throw new Error(`Google Drive token refresh failed: ${data.error_description || data.error || response.statusText}`);
+  const tokens = {
+    accessToken: data.access_token,
+    refreshToken: currentTokens.refreshToken,
+    expiresAt: Date.now() + (data.expires_in ?? 3600) * 1000,
+    tokenType: data.token_type ?? currentTokens.tokenType,
+    scope: data.scope ?? currentTokens.scope
+  };
+  saveTokens3(profile, tokens, preferredBaseDir);
+  return tokens;
+}
+function loadCredentials3(profile) {
+  const envClientId = process.env.GOOGLE_CLIENT_ID;
+  const envClientSecret = process.env.GOOGLE_CLIENT_SECRET;
+  if (envClientId && envClientSecret)
+    return { clientId: envClientId, clientSecret: envClientSecret };
+  for (const baseDir of configDirs3()) {
+    const credentials = {
+      ...readJson3(join8(baseDir, "credentials.json")),
+      ...readJson3(join8(baseDir, "profiles", profile, "config.json"))
+    };
+    if (credentials.clientId || credentials.clientSecret)
+      return credentials;
+  }
+  return {};
+}
+function loadTokens4(profile) {
+  for (const baseDir of configDirs3()) {
+    const fromProfile = readJson3(join8(baseDir, "profiles", profile, "tokens.json"));
+    if (fromProfile)
+      return { tokens: fromProfile, baseDir };
+    const flat = readJson3(join8(baseDir, "profiles", `${profile}.json`));
+    const tokens = flat?.tokens ?? (flat?.accessToken || flat?.refreshToken ? flat : null);
+    if (tokens)
+      return { tokens, baseDir };
+  }
+  return null;
+}
+function saveTokens3(profile, tokens, preferredBaseDir) {
+  const dirs = configDirs3();
+  const baseDir = preferredBaseDir ?? dirs.find((dir) => existsSync9(dir)) ?? dirs[0];
+  if (!baseDir)
+    throw new Error("Google Drive connector configuration directory is unavailable");
+  const profileDir = join8(baseDir, "profiles", profile);
+  mkdirSync8(profileDir, { recursive: true });
+  writeFileSync7(join8(profileDir, "tokens.json"), JSON.stringify(tokens, null, 2), { mode: 384 });
+}
+function configDirs3() {
+  const explicit = process.env.HASNA_GOOGLE_DRIVE_CONNECTOR_DIR ?? process.env.GOOGLE_DRIVE_CONNECTOR_DIR;
+  if (explicit)
+    return [explicit];
+  const baseDir = process.env.HASNA_CONNECTORS_DIR ?? join8(homedir7(), ".hasna", "connectors");
+  return [join8(baseDir, "googledrive"), join8(baseDir, "connect-googledrive")];
+}
+function readJson3(path) {
+  if (!existsSync9(path))
+    return null;
+  try {
+    return JSON.parse(readFileSync8(path, "utf8"));
+  } catch {
+    return null;
+  }
+}
+function extractGoogleError2(body) {
+  if (!body)
+    return "";
+  try {
+    const parsed = JSON.parse(body);
+    if (typeof parsed.error === "object" && parsed.error?.message)
+      return parsed.error.message;
+    if (typeof parsed.error === "string")
+      return parsed.error_description || parsed.error;
+  } catch {}
+  return body.slice(0, 500);
+}
 
 // src/lib/google-drive.ts
 var DRIVE_FIELDS = "nextPageToken,files(id,name,mimeType,size,modifiedTime,parents,version,md5Checksum)";
+var STREAM_TO_S3_THRESHOLD_BYTES = 64 * 1024 * 1024;
 var clientFactory = createConnectorProfileGoogleDriveClient;
 var profileStatusProvider = listGoogleDriveProfileStatusesFromConnectorConfig;
 var storageAdapter = {
@@ -167487,9 +167666,17 @@ var storageAdapter = {
   writeLocal: async (source, relativePath, data) => {
     if (!source.path)
       throw new Error("Local destination source missing path");
-    const localPath = join8(source.path, relativePath);
-    mkdirSync8(dirname5(localPath), { recursive: true });
-    writeFileSync7(localPath, data);
+    const localPath = join11(source.path, relativePath);
+    mkdirSync10(dirname5(localPath), { recursive: true });
+    writeFileSync9(localPath, data);
+    return relativePath;
+  },
+  writeLocalStream: async (source, relativePath, body) => {
+    if (!source.path)
+      throw new Error("Local destination source missing path");
+    const localPath = join11(source.path, relativePath);
+    mkdirSync10(dirname5(localPath), { recursive: true });
+    await pipeline2(body, createWriteStream2(localPath));
     return relativePath;
   }
 };
@@ -167615,22 +167802,17 @@ async function syncGoogleDriveSource(source) {
         const existing = getGoogleDriveImportedObject(source.id, item.drive_id, item.id);
         if (existing && !shouldImport(config9, item, existing, destination.source.id, destination.storage_type))
           continue;
-        const downloaded = await downloadOrArchiveGoogleDriveItem(client2, item, config9);
-        const importedName = basename5(downloaded.filename);
-        const importedPath = buildImportedPath(config9, item, importedName);
-        const contentType = downloaded.mimeType || ($lookup(downloaded.filename) || item.mime || "application/octet-stream");
-        const data = Buffer.from(downloaded.data);
-        const storageKey = await writeToDestination(destination.source, destination.storage_type, importedPath, data, contentType);
+        const importResult = await importGoogleDriveItem(client2, item, config9, destination.source, destination.storage_type);
         const fileRecord = upsertFile({
           id: existing?.file_record_id,
           source_id: destination.source.id,
           machine_id: machine.id,
-          path: storageKey,
-          name: importedName,
-          ext: extname5(importedName).toLowerCase(),
-          size: data.byteLength,
-          mime: contentType,
-          hash: item.hash ?? hashBuffer(data),
+          path: importResult.storageKey,
+          name: importResult.importedName,
+          ext: extname5(importResult.importedName).toLowerCase(),
+          size: importResult.size,
+          mime: importResult.contentType,
+          hash: importResult.hash,
           status: "active",
           modified_at: item.modified_at
         });
@@ -167640,17 +167822,17 @@ async function syncGoogleDriveSource(source) {
           file_id: item.id,
           profile: config9.profile,
           parent_id: item.parent_id,
-          path: importedPath,
-          name: importedName,
-          mime: contentType,
-          size: data.byteLength,
+          path: importResult.importedPath,
+          name: importResult.importedName,
+          mime: importResult.contentType,
+          size: importResult.size,
           modified_at: item.modified_at,
           version: item.version,
-          hash: item.hash,
+          hash: importResult.hash,
           storage_type: destination.storage_type,
-          storage_key: storageKey,
+          storage_key: importResult.storageKey,
           destination_source_id: destination.source.id,
-          s3_key: destination.storage_type === "s3" ? storageKey : "",
+          s3_key: destination.storage_type === "s3" ? importResult.storageKey : "",
           file_record_id: fileRecord.id,
           deleted: false,
           last_imported_at: new Date().toISOString()
@@ -167715,6 +167897,51 @@ async function writeToDestination(source, storageType, importedPath, data, conte
   }
   return storageAdapter.writeLocal(source, importedPath, data);
 }
+async function importGoogleDriveItem(client2, item, config9, destinationSource, storageType) {
+  if (shouldStreamGoogleDriveItem(client2, item, storageType)) {
+    const downloaded2 = await client2.downloadFileStream(toApiFile(item), config9.export_formats);
+    return importGoogleDriveStream(destinationSource, storageType, item, config9, downloaded2);
+  }
+  const downloaded = await downloadOrArchiveGoogleDriveItem(client2, item, config9);
+  const importedName = basename5(downloaded.filename);
+  const importedPath = buildImportedPath(config9, item, importedName);
+  const contentType = downloaded.mimeType || ($lookup(downloaded.filename) || item.mime || "application/octet-stream");
+  const data = Buffer.from(downloaded.data);
+  const storageKey = await writeToDestination(destinationSource, storageType, importedPath, data, contentType);
+  return {
+    importedName,
+    importedPath,
+    contentType,
+    size: data.byteLength,
+    hash: item.hash ?? hashBuffer(data),
+    storageKey
+  };
+}
+function shouldStreamGoogleDriveItem(client2, item, storageType) {
+  return storageType === "s3" && typeof client2.downloadFileStream === "function" && !item.mime.startsWith("application/vnd.google-apps.") && item.size >= STREAM_TO_S3_THRESHOLD_BYTES;
+}
+async function importGoogleDriveStream(source, storageType, item, config9, downloaded) {
+  const importedName = basename5(downloaded.filename);
+  const importedPath = buildImportedPath(config9, item, importedName);
+  const contentType = downloaded.mimeType || ($lookup(downloaded.filename) || item.mime || "application/octet-stream");
+  const size = downloaded.size ?? item.size;
+  const storageKey = await writeStreamToDestination(source, storageType, importedPath, downloaded.body, contentType, size);
+  return {
+    importedName,
+    importedPath,
+    contentType,
+    size,
+    hash: item.hash,
+    storageKey
+  };
+}
+async function writeStreamToDestination(source, storageType, importedPath, body, contentType, contentLength) {
+  if (storageType === "s3") {
+    const key = buildStorageKey(source, importedPath);
+    return storageAdapter.uploadS3(source, body, key, contentType, contentLength);
+  }
+  return storageAdapter.writeLocalStream(source, importedPath, body);
+}
 async function getIncludedSharedDrives(source, client2) {
   const config9 = getGoogleDriveConfig(source);
   if (!config9.include_all_shared_drives && (!config9.shared_drive_ids || config9.shared_drive_ids.length === 0)) {
@@ -167979,8 +168206,8 @@ function requireId(partial, table) {
 }
 
 // src/cli/index.tsx
-import { resolve as resolve3, join as join17 } from "path";
-import { existsSync as existsSync9, readFileSync as readFileSync8 } from "fs";
+import { resolve as resolve3, join as join18 } from "path";
+import { existsSync as existsSync13, readFileSync as readFileSync10 } from "fs";
 import { createRequire as createRequire2 } from "module";
 var _require = createRequire2(import.meta.url);
 var _pkg = _require("../../package.json");
@@ -168037,7 +168264,7 @@ sources.command("add <path-or-s3>").description("Add a local folder or S3 bucket
     console.log(chalk.green(`\u2713 S3 source added: ${source.id} \u2192 s3://${bucket}${prefix ? `/${prefix}` : ""}`));
   } else {
     const absPath = resolve3(pathOrS3);
-    if (!existsSync9(absPath)) {
+    if (!existsSync13(absPath)) {
       console.error(chalk.red(`Path does not exist: ${absPath}`));
       process.exit(1);
     }
@@ -168478,7 +168705,7 @@ program2.command("download <file-id> [dest]").description("Download a file to lo
     process.exit(1);
   }
   if (source.type === "local") {
-    const fullPath = join17(source.path, file.path);
+    const fullPath = join18(source.path, file.path);
     console.log(chalk.dim(`Local file at: ${fullPath}`));
     return;
   }
@@ -168499,7 +168726,7 @@ program2.command("upload <local-path> <source-id> [s3-key]").description("Upload
     console.error(chalk.red("upload only works with S3 sources"));
     process.exit(1);
   }
-  if (!existsSync9(localPath)) {
+  if (!existsSync13(localPath)) {
     console.error(chalk.red(`File not found: ${localPath}`));
     process.exit(1);
   }
@@ -168772,7 +168999,7 @@ program2.command("open <file-id>").description("Open a file in the default appli
       console.error(chalk.red("open only works with local sources"));
       process.exit(1);
     }
-    const fullPath = join17(source.path, file.path);
+    const fullPath = join18(source.path, file.path);
     Bun.spawn(getOpenCommand(fullPath), { stdout: "inherit", stderr: "inherit" });
   } catch (e3) {
     console.error(chalk.red(e3.message));
@@ -168787,7 +169014,7 @@ program2.command("where <file-id>").description("Print the full absolute path of
       console.error(chalk.red("where only works with local sources"));
       process.exit(1);
     }
-    process.stdout.write(join17(source.path, file.path) + `
+    process.stdout.write(join18(source.path, file.path) + `
 `);
   } catch (e3) {
     console.error(chalk.red(e3.message));
@@ -168802,9 +169029,9 @@ program2.command("cat <file-id>").description("Print file content to stdout").op
       console.error(chalk.red("cat only works with local sources"));
       process.exit(1);
     }
-    const fullPath = join17(source.path, file.path);
+    const fullPath = join18(source.path, file.path);
     const maxBytes = parseIntFlag(opts.maxBytes, "max-bytes", { min: 0 });
-    const buf = readFileSync8(fullPath);
+    const buf = readFileSync10(fullPath);
     const slice = maxBytes > 0 ? buf.slice(0, maxBytes) : buf;
     process.stdout.write(slice);
   } catch (e3) {