@jackwener/opencli 1.3.3 → 1.4.1

package/src/clis/douyin/_shared/tos-upload.ts

@@ -0,0 +1,444 @@
+/**
+ * TOS (ByteDance Object Storage) multipart uploader with resume support.
+ *
+ * Uses AWS Signature V4 (HMAC-SHA256) with STS2 temporary credentials.
+ * For the init multipart upload call, the pre-computed auth from TosUploadInfo is used.
+ * For PUT part uploads and the final complete call, AWS4 is computed from STS2 credentials.
+ */
+
+import * as crypto from 'node:crypto';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { CommandExecutionError } from '../../../errors.js';
+import type { Sts2Credentials, TosUploadInfo } from './types.js';
+
+export interface TosUploadOptions {
+  filePath: string;
+  uploadInfo: TosUploadInfo;
+  credentials: Sts2Credentials;
+  onProgress?: (uploaded: number, total: number) => void;
+}
+
+interface ResumePart {
+  partNumber: number;
+  etag: string;
+}
+
+interface ResumeState {
+  uploadId: string;
+  fileSize: number;
+  parts: ResumePart[];
+}
+
+const PART_SIZE = 5 * 1024 * 1024; // 5 MB minimum per TOS/S3 spec
+const RESUME_DIR = path.join(os.homedir(), '.opencli', 'douyin-resume');
+
+// ── Resume file helpers ──────────────────────────────────────────────────────
+
+function getResumeFilePath(filePath: string): string {
+  const hash = crypto.createHash('sha256').update(filePath).digest('hex');
+  return path.join(RESUME_DIR, `${hash}.json`);
+}
+
+function loadResumeState(resumePath: string, fileSize: number): ResumeState | null {
+  try {
+    const raw = fs.readFileSync(resumePath, 'utf8');
+    const state = JSON.parse(raw) as ResumeState;
+    if (state.fileSize === fileSize && state.uploadId && Array.isArray(state.parts)) {
+      return state;
+    }
+  } catch {
+    // no valid resume state
+  }
+  return null;
+}
+
+function saveResumeState(resumePath: string, state: ResumeState): void {
+  fs.mkdirSync(path.dirname(resumePath), { recursive: true });
+  fs.writeFileSync(resumePath, JSON.stringify(state, null, 2), 'utf8');
+}
+
+function deleteResumeState(resumePath: string): void {
+  try {
+    fs.unlinkSync(resumePath);
+  } catch {
+    // ignore if not found
+  }
+}
+
+// ── AWS Signature V4 ─────────────────────────────────────────────────────────
+
+function hmacSha256(key: Buffer | string, data: string): Buffer {
+  return crypto.createHmac('sha256', key).update(data, 'utf8').digest();
+}
+
+function sha256Hex(data: Buffer | string): string {
+  const hash = crypto.createHash('sha256');
+  if (typeof data === 'string') {
+    hash.update(data, 'utf8');
+  } else {
+    hash.update(data);
+  }
+  return hash.digest('hex');
+}
+
+function extractRegionFromHost(host: string): string {
+  // e.g. "tos-cn-i-alisg.volces.com" → "cn-i-alisg"
+  // e.g. "tos-cn-beijing.ivolces.com" → "cn-beijing"
+  const match = host.match(/^tos-([^.]+)\./);
+  if (match) return match[1];
+  return 'cn-north-1'; // fallback
+}
+
+interface SignedHeaders {
+  [key: string]: string;
+}
+
+/**
+ * Compute AWS Signature V4 headers for a TOS request.
+ * Returns a Record of all headers to include (including Authorization, x-amz-date, etc.)
+ */
+function computeAws4Headers(opts: {
+  method: string;
+  url: string;
+  headers: Record<string, string>;
+  body: Buffer | string;
+  credentials: Sts2Credentials;
+  service: string;
+  region: string;
+  datetime: string; // YYYYMMDDTHHmmssZ
+}): SignedHeaders {
+  const { method, url, credentials, service, region, datetime } = opts;
+  const date = datetime.slice(0, 8); // YYYYMMDD
+
+  const parsedUrl = new URL(url);
+  const canonicalUri = parsedUrl.pathname || '/';
+  // Canonical query string: sort by name, encode
+  const queryParams = [...parsedUrl.searchParams.entries()]
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
+    .join('&');
+
+  const bodyHash = sha256Hex(opts.body);
+
+  // Merge in required headers and compute canonical headers
+  const allHeaders: Record<string, string> = {
+    ...opts.headers,
+    host: parsedUrl.host,
+    'x-amz-content-sha256': bodyHash,
+    'x-amz-date': datetime,
+    'x-amz-security-token': credentials.session_token,
+  };
+
+  const sortedHeaderKeys = Object.keys(allHeaders).sort((a, b) =>
+    a.toLowerCase().localeCompare(b.toLowerCase())
+  );
+
+  const canonicalHeaders = sortedHeaderKeys
+    .map(k => `${k.toLowerCase()}:${allHeaders[k].trim()}`)
+    .join('\n') + '\n';
+
+  const signedHeadersList = sortedHeaderKeys.map(k => k.toLowerCase()).join(';');
+
+  const canonicalRequest = [
+    method.toUpperCase(),
+    canonicalUri,
+    queryParams,
+    canonicalHeaders,
+    signedHeadersList,
+    bodyHash,
+  ].join('\n');
+
+  const credentialScope = `${date}/${region}/${service}/aws4_request`;
+  const stringToSign = [
+    'AWS4-HMAC-SHA256',
+    datetime,
+    credentialScope,
+    sha256Hex(canonicalRequest),
+  ].join('\n');
+
+  // Signing key chain
+  const kDate = hmacSha256(`AWS4${credentials.secret_access_key}`, date);
+  const kRegion = hmacSha256(kDate, region);
+  const kService = hmacSha256(kRegion, service);
+  const kSigning = hmacSha256(kService, 'aws4_request');
+  const signature = hmacSha256(kSigning, stringToSign).toString('hex');
+
+  const authorization = `AWS4-HMAC-SHA256 Credential=${credentials.access_key_id}/${credentialScope}, SignedHeaders=${signedHeadersList}, Signature=${signature}`;
+
+  return {
+    ...allHeaders,
+    Authorization: authorization,
+  };
+}
+
+// ── HTTP helpers ─────────────────────────────────────────────────────────────
+
+async function tosRequest(opts: {
+  method: string;
+  url: string;
+  headers: Record<string, string>;
+  body?: Buffer | string;
+}): Promise<{ status: number; headers: Record<string, string>; body: string }> {
+  const { method, url, headers, body } = opts;
+  const fetchBody: BodyInit | null =
+    body == null ? null
+    : typeof body === 'string' ? body
+    : (body as unknown as Uint8Array<ArrayBuffer>);
+  const res = await fetch(url, {
+    method,
+    headers,
+    body: fetchBody,
+  });
+  const responseBody = await res.text();
+  const responseHeaders: Record<string, string> = {};
+  res.headers.forEach((value, key) => {
+    responseHeaders[key.toLowerCase()] = value;
+  });
+  return { status: res.status, headers: responseHeaders, body: responseBody };
+}
+
+function nowDatetime(): string {
+  return new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
+}
+
+// ── Phase 1: Init multipart upload ───────────────────────────────────────────
+
+async function initMultipartUpload(
+  tosUrl: string,
+  auth: string,
+  credentials: Sts2Credentials,
+): Promise<string> {
+  const initUrl = `${tosUrl}?uploads`;
+  const datetime = nowDatetime();
+
+  // Use the pre-computed auth for INIT, as it comes from ApplyVideoUpload
+  const headers: Record<string, string> = {
+    Authorization: auth,
+    'x-amz-date': datetime,
+    'x-amz-security-token': credentials.session_token,
+    'content-type': 'application/octet-stream',
+  };
+
+  const res = await tosRequest({ method: 'POST', url: initUrl, headers });
+
+  if (res.status !== 200) {
+    throw new CommandExecutionError(
+      `TOS init multipart upload failed with status ${res.status}: ${res.body}`,
+      'Check that TOS credentials are valid and not expired.',
+    );
+  }
+
+  // Parse UploadId from XML: <UploadId>...</UploadId>
+  const match = res.body.match(/<UploadId>([^<]+)<\/UploadId>/);
+  if (!match) {
+    throw new CommandExecutionError(
+      `TOS init response missing UploadId: ${res.body}`,
+    );
+  }
+  return match[1];
+}
+
+// ── Phase 2: Upload a single part ────────────────────────────────────────────
+
+async function uploadPart(
+  tosUrl: string,
+  partNumber: number,
+  uploadId: string,
+  data: Buffer,
+  credentials: Sts2Credentials,
+  region: string,
+): Promise<string> {
+  const parsedUrl = new URL(tosUrl);
+  parsedUrl.searchParams.set('partNumber', String(partNumber));
+  parsedUrl.searchParams.set('uploadId', uploadId);
+  const url = parsedUrl.toString();
+
+  const datetime = nowDatetime();
+  const headers = computeAws4Headers({
+    method: 'PUT',
+    url,
+    headers: { 'content-type': 'application/octet-stream' },
+    body: data,
+    credentials,
+    service: 'tos',
+    region,
+    datetime,
+  });
+
+  const res = await tosRequest({ method: 'PUT', url, headers, body: data });
+
+  if (res.status !== 200) {
+    throw new CommandExecutionError(
+      `TOS upload part ${partNumber} failed with status ${res.status}: ${res.body}`,
+      'Check that STS2 credentials are valid and not expired.',
+    );
+  }
+
+  const etag = res.headers['etag'];
+  if (!etag) {
+    throw new CommandExecutionError(
+      `TOS upload part ${partNumber} response missing ETag header`,
+    );
+  }
+  return etag;
+}
+
+// ── Phase 3: Complete multipart upload ───────────────────────────────────────
+
+async function completeMultipartUpload(
+  tosUrl: string,
+  uploadId: string,
+  parts: ResumePart[],
+  credentials: Sts2Credentials,
+  region: string,
+): Promise<void> {
+  const parsedUrl = new URL(tosUrl);
+  parsedUrl.searchParams.set('uploadId', uploadId);
+  const url = parsedUrl.toString();
+
+  const xmlBody =
+    '<CompleteMultipartUpload>' +
+    parts
+      .sort((a, b) => a.partNumber - b.partNumber)
+      .map(p => `<Part><PartNumber>${p.partNumber}</PartNumber><ETag>${p.etag}</ETag></Part>`)
+      .join('') +
+    '</CompleteMultipartUpload>';
+
+  const datetime = nowDatetime();
+  const headers = computeAws4Headers({
+    method: 'POST',
+    url,
+    headers: { 'content-type': 'application/xml' },
+    body: xmlBody,
+    credentials,
+    service: 'tos',
+    region,
+    datetime,
+  });
+
+  const res = await tosRequest({
+    method: 'POST',
+    url,
+    headers,
+    body: xmlBody,
+  });
+
+  if (res.status !== 200) {
+    throw new CommandExecutionError(
+      `TOS complete multipart upload failed with status ${res.status}: ${res.body}`,
+      'Check that all parts were uploaded successfully.',
+    );
+  }
+}
+
+// ── Testing seams ────────────────────────────────────────────────────────────
+
+// Allow tests to override the readSync behaviour (e.g. to simulate short reads)
+// without fighting ESM live-binding limitations.
+type ReadSyncFn = (fd: number, buffer: Buffer, offset: number, length: number, position: number) => number;
+let _readSyncOverride: ReadSyncFn | null = null;
+
+/** @internal — for testing only */
+export function setReadSyncOverride(fn: ReadSyncFn | null): void {
+  _readSyncOverride = fn;
+}
+
+// ── Public API ───────────────────────────────────────────────────────────────
+
+export async function tosUpload(options: TosUploadOptions): Promise<void> {
+  const { filePath, uploadInfo, credentials, onProgress } = options;
+
+  // Validate file exists
+  if (!fs.existsSync(filePath)) {
+    throw new CommandExecutionError(
+      `Video file not found: ${filePath}`,
+      'Ensure the file path is correct and accessible.',
+    );
+  }
+
+  const { size: fileSize } = fs.statSync(filePath);
+  if (fileSize === 0) {
+    throw new CommandExecutionError(`Video file is empty: ${filePath}`);
+  }
+
+  const { tos_upload_url: tosUrl, auth } = uploadInfo;
+  const parsedTosUrl = new URL(tosUrl);
+  const region = extractRegionFromHost(parsedTosUrl.host);
+
+  const resumePath = getResumeFilePath(filePath);
+  let resumeState = loadResumeState(resumePath, fileSize);
+
+  let uploadId: string;
+  let completedParts: ResumePart[];
+
+  if (resumeState) {
+    // Resume from previous state
+    uploadId = resumeState.uploadId;
+    completedParts = resumeState.parts;
+  } else {
+    // Start fresh
+    uploadId = await initMultipartUpload(tosUrl, auth, credentials);
+    completedParts = [];
+    saveResumeState(resumePath, { uploadId, fileSize, parts: completedParts });
+  }
+
+  // Determine which parts are already done
+  const completedPartNumbers = new Set(completedParts.map(p => p.partNumber));
+
+  // Calculate total parts
+  const totalParts = Math.ceil(fileSize / PART_SIZE);
+
+  // Track uploaded bytes for progress
+  let uploadedBytes = completedParts.length * PART_SIZE;
+  if (onProgress) onProgress(Math.min(uploadedBytes, fileSize), fileSize);
+
+  const fd = fs.openSync(filePath, 'r');
+  try {
+    for (let partNumber = 1; partNumber <= totalParts; partNumber++) {
+      if (completedPartNumbers.has(partNumber)) {
+        continue; // already uploaded
+      }
+
+      const offset = (partNumber - 1) * PART_SIZE;
+      const chunkSize = Math.min(PART_SIZE, fileSize - offset);
+      const buffer = Buffer.allocUnsafe(chunkSize);
+      const readFn = _readSyncOverride ?? fs.readSync;
+      const bytesRead = readFn(fd, buffer, 0, chunkSize, offset);
+      if (bytesRead !== chunkSize) {
+        throw new CommandExecutionError(
+          `Short read on part ${partNumber}: expected ${chunkSize} bytes, got ${bytesRead}`,
+        );
+      }
+
+      const etag = await uploadPart(tosUrl, partNumber, uploadId, buffer, credentials, region);
+
+      completedParts.push({ partNumber, etag });
+      saveResumeState(resumePath, { uploadId, fileSize, parts: completedParts });
+
+      uploadedBytes = Math.min(offset + chunkSize, fileSize);
+      if (onProgress) onProgress(uploadedBytes, fileSize);
+    }
+  } finally {
+    fs.closeSync(fd);
+  }
+
+  await completeMultipartUpload(tosUrl, uploadId, completedParts, credentials, region);
+
+  deleteResumeState(resumePath);
+}
+
+// ── Internal exports for testing ─────────────────────────────────────────────
+
+export {
+  PART_SIZE,
+  RESUME_DIR,
+  extractRegionFromHost,
+  getResumeFilePath,
+  loadResumeState,
+  saveResumeState,
+  deleteResumeState,
+  computeAws4Headers,
+};
+export type { ResumeState, ResumePart };

package/src/clis/douyin/_shared/transcode.test.ts

@@ -0,0 +1,117 @@
+import { describe, expect, it, vi } from 'vitest';
+import type { IPage } from '../../../types.js';
+import { TimeoutError } from '../../../errors.js';
+import { pollTranscodeWithFetch } from './transcode.js';
+import type { TranscodeResult } from './types.js';
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+function makePage(): IPage {
+  return {
+    goto: vi.fn(),
+    evaluate: vi.fn(),
+    getCookies: vi.fn(),
+    snapshot: vi.fn(),
+    click: vi.fn(),
+    typeText: vi.fn(),
+    pressKey: vi.fn(),
+    scrollTo: vi.fn(),
+    getFormState: vi.fn(),
+    wait: vi.fn(),
+    tabs: vi.fn(),
+    closeTab: vi.fn(),
+    newTab: vi.fn(),
+    selectTab: vi.fn(),
+    networkRequests: vi.fn(),
+    consoleMessages: vi.fn(),
+    scroll: vi.fn(),
+    autoScroll: vi.fn(),
+    installInterceptor: vi.fn(),
+    getInterceptedRequests: vi.fn(),
+    screenshot: vi.fn(),
+  } as unknown as IPage;
+}
+
+const COMPLETE_RESULT: TranscodeResult & { encode: number } = {
+  encode: 2,
+  duration: 30,
+  fps: 30,
+  height: 1920,
+  width: 1080,
+  poster_uri: 'tos-cn-i-alisg.volces.com/poster/abc',
+  poster_url: 'https://p3-creator.douyinpic.com/poster/abc',
+};
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe('pollTranscodeWithFetch', () => {
+  it('returns TranscodeResult immediately when first response has encode=2', async () => {
+    const fetchFn = vi.fn().mockResolvedValue(COMPLETE_RESULT);
+    const page = makePage();
+
+    const result = await pollTranscodeWithFetch(fetchFn, page, 'vid_123');
+
+    expect(result).toEqual(COMPLETE_RESULT);
+    expect(fetchFn).toHaveBeenCalledOnce();
+    const [, method, url] = fetchFn.mock.calls[0] as [IPage, string, string];
+    expect(method).toBe('GET');
+    expect(url).toContain('video_id=vid_123');
+    expect(url).toContain('aid=1128');
+  });
+
+  it('polls multiple times until encode=2 is received', async () => {
+    const pending = { encode: 1, duration: 0, fps: 0, height: 0, width: 0, poster_uri: '', poster_url: '' };
+    const fetchFn = vi
+      .fn()
+      .mockResolvedValueOnce(pending)
+      .mockResolvedValueOnce(pending)
+      .mockResolvedValueOnce(COMPLETE_RESULT);
+    const page = makePage();
+
+    // Use a large timeoutMs so it doesn't expire, but override POLL_INTERVAL via
+    // a short timeout knowing we'll get 3 calls quickly with mocked promises.
+    // Since pollTranscodeWithFetch uses setTimeout for 3s between polls, we need
+    // to mock timers to keep tests fast.
+    vi.useFakeTimers();
+
+    const resultPromise = pollTranscodeWithFetch(fetchFn, page, 'vid_456', 60_000);
+
+    // Advance timers for each pending poll cycle
+    await vi.runAllTimersAsync();
+
+    const result = await resultPromise;
+    expect(result).toEqual(COMPLETE_RESULT);
+    expect(fetchFn).toHaveBeenCalledTimes(3);
+
+    vi.useRealTimers();
+  });
+
+  it('throws TimeoutError when encode never becomes 2 within timeoutMs', async () => {
+    const pending = { encode: 1, duration: 0, fps: 0, height: 0, width: 0, poster_uri: '', poster_url: '' };
+    const fetchFn = vi.fn().mockResolvedValue(pending);
+    const page = makePage();
+
+    vi.useFakeTimers();
+
+    const resultPromise = pollTranscodeWithFetch(fetchFn, page, 'vid_789', 5_000);
+    // Suppress unhandled-rejection so vitest doesn't flag it
+    resultPromise.catch(() => undefined);
+
+    // Advance time past the 5s timeout
+    await vi.runAllTimersAsync();
+
+    await expect(resultPromise).rejects.toBeInstanceOf(TimeoutError);
+
+    vi.useRealTimers();
+  });
+
+  it('URL encodes video_id in the request URL', async () => {
+    const fetchFn = vi.fn().mockResolvedValue(COMPLETE_RESULT);
+    const page = makePage();
+
+    await pollTranscodeWithFetch(fetchFn, page, 'vid with spaces');
+
+    const [, , url] = fetchFn.mock.calls[0] as [IPage, string, string];
+    expect(url).toContain('video_id=vid%20with%20spaces');
+  });
+});

package/src/clis/douyin/_shared/transcode.ts

@@ -0,0 +1,78 @@
+/**
+ * Transcode poller for Douyin video processing.
+ *
+ * After a video is uploaded via TOS and the "confirm upload" API is called,
+ * Douyin transcodes the video asynchronously. This module polls the transcode
+ * status endpoint until encode=2 (complete) or a timeout is reached.
+ */
+
+import type { IPage } from '../../../types.js';
+import { TimeoutError } from '../../../errors.js';
+import { browserFetch } from './browser-fetch.js';
+import type { TranscodeResult } from './types.js';
+
+const POLL_INTERVAL_MS = 3_000;
+const DEFAULT_TIMEOUT_MS = 300_000;
+
+const TRANSCODE_URL_BASE =
+  'https://creator.douyin.com/web/api/media/video/transend/';
+
+type BrowserFetchFn = (
+  page: IPage,
+  method: 'GET' | 'POST',
+  url: string,
+) => Promise<unknown>;
+
+/**
+ * Lower-level poll function that accepts an injected fetch function.
+ * Exported for testability.
+ */
+export async function pollTranscodeWithFetch(
+  fetchFn: BrowserFetchFn,
+  page: IPage,
+  videoId: string,
+  timeoutMs = DEFAULT_TIMEOUT_MS,
+): Promise<TranscodeResult> {
+  const url = `${TRANSCODE_URL_BASE}?video_id=${encodeURIComponent(videoId)}&aid=1128`;
+  const deadline = Date.now() + timeoutMs;
+
+  while (Date.now() < deadline) {
+    const result = (await fetchFn(page, 'GET', url)) as {
+      encode: number;
+    } & TranscodeResult;
+
+    if (result.encode === 2) {
+      return result;
+    }
+
+    // Wait before next poll, but don't exceed the deadline
+    const remaining = deadline - Date.now();
+    if (remaining <= 0) break;
+    await new Promise<void>(resolve =>
+      setTimeout(resolve, Math.min(POLL_INTERVAL_MS, remaining)),
+    );
+  }
+
+  throw new TimeoutError(
+    `Douyin transcode for video ${videoId}`,
+    Math.round(timeoutMs / 1000),
+  );
+}
+
+/**
+ * Poll Douyin's transcode status endpoint until the video is fully transcoded
+ * (encode=2) or the timeout expires.
+ *
+ * @param page - Browser page for making credentialed API calls
+ * @param videoId - The video_id returned from the confirm upload step
+ * @param timeoutMs - Maximum wait time in ms (default: 300 000 = 5 minutes)
+ * @returns TranscodeResult including duration, fps, dimensions, and poster info
+ * @throws TimeoutError if transcode does not complete within timeoutMs
+ */
+export async function pollTranscode(
+  page: IPage,
+  videoId: string,
+  timeoutMs?: number,
+): Promise<TranscodeResult> {
+  return pollTranscodeWithFetch(browserFetch, page, videoId, timeoutMs);
+}

package/src/clis/douyin/_shared/types.ts

@@ -0,0 +1,29 @@
+export interface Sts2Credentials {
+  access_key_id: string;
+  secret_access_key: string;
+  session_token: string;
+  expired_time: number;
+}
+
+export interface TosUploadInfo {
+  tos_upload_url: string;
+  /** Pre-computed Authorization header value returned by ApplyVideoUpload (StoreInfos[0].Auth) */
+  auth: string;
+  video_id: string;
+}
+
+export interface TranscodeResult {
+  encode: number;
+  duration: number;
+  fps: number;
+  height: number;
+  width: number;
+  poster_uri: string;
+  poster_url: string;
+}
+
+export interface PublishResult {
+  aweme_id: string;
+  url: string;
+  publish_time: number;
+}

package/src/clis/douyin/activities.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from 'vitest';
+import { getRegistry } from '../../registry.js';
+import './activities.js';
+
+describe('douyin activities registration', () => {
+  it('registers the activities command', () => {
+    const registry = getRegistry();
+    const cmd = [...registry.values()].find(c => c.site === 'douyin' && c.name === 'activities');
+    expect(cmd).toBeDefined();
+  });
+
+  it('has expected columns', () => {
+    const registry = getRegistry();
+    const cmd = [...registry.values()].find(c => c.site === 'douyin' && c.name === 'activities');
+    expect(cmd?.columns).toContain('activity_id');
+    expect(cmd?.columns).toContain('title');
+    expect(cmd?.columns).toContain('end_time');
+  });
+
+  it('uses COOKIE strategy', () => {
+    const registry = getRegistry();
+    const cmd = [...registry.values()].find(c => c.site === 'douyin' && c.name === 'activities');
+    expect(cmd?.strategy).toBe('cookie');
+  });
+});