@chilfish/gallery-dl-instagram 0.1.0

package/cli/adapter.ts

@@ -0,0 +1,284 @@
+/**
+ * Node.js adapter — HttpClient + Storage + Logger implementations.
+ *
+ * These bind the platform-agnostic SDK interfaces to Node.js primitives:
+ * axios for HTTP, fs/promises for file I/O, console for logging.
+ */
+
+import type { AxiosInstance } from 'axios'
+import type { Logger } from '../core/extractor'
+import type { HttpClient, HttpResponse, Storage } from '../types'
+import { access, mkdir, writeFile } from 'node:fs/promises'
+import { dirname } from 'node:path'
+import axios from 'axios'
+import { createCookieJar } from './cookies'
+
+/** NodeHttpClient — axios wrapper */
+
+/**
+ * Extract csrftoken value from a Cookie header string.
+ */
+export function extractCsrfFromCookies(cookies: string): string {
+  const m = cookies.match(/(?:^|;\s*)csrftoken=([^;]+)/)
+  return m?.[1] ?? ''
+}
+
+export function createHttpClient(
+  sessionId?: string,
+  fullCookies?: string,
+  logger?: Logger,
+): HttpClient {
+  const instance: AxiosInstance = axios.create({
+    timeout: 30000,
+    maxRedirects: 20,
+    validateStatus: () => true, // don't throw on non-2xx
+    headers: {
+      'User-Agent':
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    },
+  })
+
+  // Build base Cookie string
+  // Priority: full cookies string > sessionid-only
+  const baseCookie = fullCookies
+    || (sessionId ? `sessionid=${sessionId}` : null)
+
+  return {
+    async request<T = unknown>(config: {
+      url: string
+      method?: string
+      headers?: Record<string, string>
+      params?: Record<string, string | number | null | undefined>
+      data?: unknown
+      signal?: AbortSignal
+      timeout?: number
+      responseType?: 'arraybuffer' | 'text' | 'json'
+    }): Promise<HttpResponse<T>> {
+      const method = config.method ?? 'GET'
+      logger?.debug(`${method} ${config.url}`)
+
+      // Merge base cookie with per-request headers (Cookie values are joined)
+      const mergedHeaders: Record<string, string> = {}
+      if (baseCookie) {
+        mergedHeaders.Cookie = baseCookie
+      }
+      if (config.headers) {
+        for (const [k, v] of Object.entries(config.headers)) {
+          if (k.toLowerCase() === 'cookie' && mergedHeaders.Cookie) {
+            // Append instead of overriding: sessionid=xxx + csrftoken=yyy → sessionid=xxx; csrftoken=yyy
+            mergedHeaders.Cookie = `${mergedHeaders.Cookie}; ${v}`
+          }
+          else {
+            mergedHeaders[k] = v
+          }
+        }
+      }
+
+      if (mergedHeaders.Cookie) {
+        logger?.debug(`  Cookie: ${mergedHeaders.Cookie.slice(0, 200)}`)
+      }
+
+      try {
+        const resp = await instance.request<T>({
+          url: config.url,
+          method,
+          headers: mergedHeaders,
+          params: cleanupParams(config.params),
+          data: config.data,
+          signal: config.signal,
+          timeout: config.timeout,
+          responseType: config.responseType ?? 'json',
+        })
+
+        const finalUrl = resp.request?.res?.responseUrl ?? config.url
+        logger?.debug(`  ← ${resp.status} ${resp.status >= 400 ? '⚠️' : ''} (${finalUrl.slice(0, 100)})`)
+
+        return {
+          status: resp.status,
+          data: resp.data,
+          headers: resp.headers as Record<string, string>,
+          url: finalUrl,
+        }
+      }
+      catch (err) {
+        const msg = String(err)
+        if (msg.includes('TOO_MANY_REDIRECTS') || msg.includes('too many redirects')) {
+          throw new Error(
+            'Too many redirects — sessionid may be expired or invalid. '
+            + 'Export a fresh sessionid from your browser.',
+          )
+        }
+        throw err
+      }
+    },
+  }
+}
+
+/** WebClient — anonymous cookie-jar HTTP client */
+
+/**
+ * Create an HTTP client with an in-memory cookie jar.
+ *
+ * Use this when you don't have a sessionid — the client first seeds its
+ * cookie jar by visiting ``instagram.com``, then uses those anonymous
+ * cookies for subsequent API calls.  This is how incognito browsing works.
+ *
+ * Returns the client + the initial CSRF token extracted from cookies.
+ */
+export async function createWebClient(
+  logger?: Logger,
+): Promise<{ http: HttpClient, csrfToken: string }> {
+  const jar = createCookieJar()
+
+  // Seed the cookie jar by visiting Instagram's homepage
+  logger?.info('Seeding anonymous session (visiting instagram.com)…')
+  const seedResp = await axios.get('https://www.instagram.com/', {
+    headers: {
+      'User-Agent':
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+      'Accept':
+        'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    },
+    maxRedirects: 20,
+    validateStatus: () => true,
+  })
+  jar.setFromResponse(seedResp.headers as Record<string, string>)
+  logger?.debug(`  ← ${seedResp.status} — got ${jar.getCookieHeader().split(';').length} cookies`)
+
+  // Extract csrftoken from jar
+  const cookieStr = jar.getCookieHeader()
+  const csrfMatch = cookieStr.match(/(?:^|;\s*)csrftoken=([^;]+)/)
+  const csrfToken = csrfMatch?.[1] ?? ''
+
+  // Build the wrapper
+  const http: HttpClient = {
+    async request<T = unknown>(config: {
+      url: string
+      method?: string
+      headers?: Record<string, string>
+      params?: Record<string, string | number | null | undefined>
+      data?: unknown
+      signal?: AbortSignal
+      timeout?: number
+      responseType?: 'arraybuffer' | 'text' | 'json'
+    }): Promise<HttpResponse<T>> {
+      const method = config.method ?? 'GET'
+      logger?.debug(`${method} ${config.url}`)
+
+      // Merge jar cookies with per-request headers (Cookie values are joined)
+      const jarCookie = jar.getCookieHeader()
+      const mergedHeaders: Record<string, string> = {}
+      if (jarCookie) {
+        mergedHeaders.Cookie = jarCookie
+      }
+      if (config.headers) {
+        for (const [k, v] of Object.entries(config.headers)) {
+          if (k.toLowerCase() === 'cookie' && mergedHeaders.Cookie) {
+            mergedHeaders.Cookie = `${mergedHeaders.Cookie}; ${v}`
+          }
+          else {
+            mergedHeaders[k] = v
+          }
+        }
+      }
+
+      try {
+        const resp = await axios.request<T>({
+          url: config.url,
+          method,
+          headers: mergedHeaders,
+          params: cleanupParams(config.params),
+          data: config.data,
+          signal: config.signal,
+          timeout: config.timeout ?? 30000,
+          maxRedirects: 20,
+          validateStatus: () => true,
+          responseType: config.responseType ?? 'json',
+        })
+
+        // Update jar with response cookies
+        jar.setFromResponse(resp.headers as Record<string, string>)
+
+        const finalUrl = resp.request?.res?.responseUrl ?? config.url
+        logger?.debug(`  ← ${resp.status} ${resp.status >= 400 ? '⚠️' : ''} (${finalUrl.slice(0, 100)})`)
+
+        return {
+          status: resp.status,
+          data: resp.data,
+          headers: resp.headers as Record<string, string>,
+          url: finalUrl,
+        }
+      }
+      catch (err) {
+        const msg = String(err)
+        if (msg.includes('TOO_MANY_REDIRECTS') || msg.includes('too many redirects')) {
+          throw new Error(
+            'Too many redirects — Instagram may be blocking the request. Try again later or use --sessionid.',
+          )
+        }
+        throw err
+      }
+    },
+  }
+
+  return { http, csrfToken }
+}
+
+function cleanupParams(
+  params?: Record<string, string | number | null | undefined>,
+): Record<string, string> | undefined {
+  if (!params)
+    return undefined
+  const cleaned: Record<string, string> = {}
+  for (const [k, v] of Object.entries(params)) {
+    if (v != null)
+      cleaned[k] = String(v)
+  }
+  return cleaned
+}
+
+/** NodeStorage — fs/promises wrapper */
+
+export function createStorage(): Storage {
+  return {
+    async exists(path: string): Promise<boolean> {
+      try {
+        await access(path)
+        return true
+      }
+      catch {
+        return false
+      }
+    },
+
+    async write(path: string, data: Uint8Array | string): Promise<void> {
+      // Ensure parent directory
+      await mkdir(dirname(path), { recursive: true })
+      await writeFile(path, data)
+    },
+
+    async mkdir(path: string): Promise<void> {
+      await mkdir(path, { recursive: true })
+    },
+  }
+}
+
+/** NodeLogger — console wrapper */
+
+export function createLogger(verbose: boolean): Logger {
+  return {
+    debug(message: string, ...args: unknown[]): void {
+      if (verbose)
+        console.debug(`[debug] ${message}`, ...args)
+    },
+    info(message: string, ...args: unknown[]): void {
+      console.info(`[info] ${message}`, ...args)
+    },
+    warn(message: string, ...args: unknown[]): void {
+      console.warn(`[warn] ${message}`, ...args)
+    },
+    error(message: string, ...args: unknown[]): void {
+      console.error(`[error] ${message}`, ...args)
+    },
+  }
+}

package/cli/cookies.ts

@@ -0,0 +1,59 @@
+/**
+ * Simple in-memory cookie jar for Node.js HTTP clients.
+ *
+ * Tracks Set-Cookie headers across requests and injects stored cookies
+ * into subsequent requests to the same domain.  This is enough for
+ * Instagram's anonymous session flow: visit homepage → get cookies →
+ * use those cookies for API calls.
+ */
+
+export interface CookieJar {
+  /** Record cookies from a response's Set-Cookie header(s). */
+  setFromResponse: (headers: Record<string, string>) => void
+  /** Get the Cookie header value for the next request. */
+  getCookieHeader: () => string
+}
+
+export function createCookieJar(): CookieJar {
+  const cookies = new Map<string, string>()
+  let cookieString = ''
+
+  return {
+    setFromResponse(headers: Record<string, string>): void {
+      const raw = headers['set-cookie']
+      if (!raw)
+        return
+
+      const cookieHeaders = Array.isArray(raw) ? raw : [raw]
+      for (const header of cookieHeaders) {
+        const parts = header.split(';')
+        if (parts.length === 0)
+          continue
+
+        const [nameValue] = parts
+        const eqIdx = nameValue!.indexOf('=')
+        if (eqIdx <= 0)
+          continue
+
+        const name = nameValue!.slice(0, eqIdx).trim()
+        const value = nameValue!.slice(eqIdx + 1).trim()
+
+        // Skip expiration by checking Max-Age=0 or Expires=epoch
+        const rest = parts.slice(1).map((s: string) => s.trim().toLowerCase())
+        if (rest.includes('max-age=0'))
+          continue
+
+        cookies.set(name, value)
+      }
+
+      // Rebuild cookie string
+      cookieString = Array.from(cookies.entries())
+        .map(([k, v]) => `${k}=${v}`)
+        .join('; ')
+    },
+
+    getCookieHeader(): string {
+      return cookieString
+    },
+  }
+}

package/cli/index.ts

@@ -0,0 +1,337 @@
+#!/usr/bin/env node
+/**
+ * gdl-instagram — CLI entry point.
+ *
+ * Usage:
+ *   gdl-instagram <url> [options]          ← auto-detect from URL
+ *   gdl-instagram tag <hashtag> [options]
+ *   gdl-instagram saved [options]
+ *
+ * Every option is self-documented via ``--help``.
+ */
+
+import { Command } from 'commander'
+import { ConfigManager } from '../config'
+import { DownloadJob, PrintJob } from '../core/job'
+import {
+  InstagramHighlightsExtractor,
+  InstagramPostExtractor,
+  InstagramSavedExtractor,
+  InstagramStoriesExtractor,
+  InstagramTagExtractor,
+  InstagramUserExtractor,
+} from '../instagram/extractors'
+import { createHttpClient, createLogger, createStorage, createWebClient, extractCsrfFromCookies } from './adapter'
+
+/** Shared options — applied to all subcommands */
+
+interface GlobalOptions {
+  sessionid?: string
+  cookies?: string
+  output?: string
+  videos?: string
+  previews?: string
+  audio?: boolean
+  maxPosts?: number
+  cursor?: string
+  orderPosts?: string
+  orderFiles?: string
+  staticVideos?: boolean
+  api?: string
+  verbose?: boolean
+  include?: string
+  split?: boolean
+  info?: boolean
+}
+
+function addSharedOptions(cmd: Command): Command {
+  return cmd
+    .option(
+      '--sessionid <cookie>',
+      'Instagram sessionid cookie value (from browser)',
+      process.env.INSTAGRAM_SESSIONID,
+    )
+    .option(
+      '--cookies <string>',
+      'Full Cookie header string from browser (DevTools → Network → Request Headers → Cookie)',
+      process.env.INSTAGRAM_COOKIES,
+    )
+    .option('-o, --output <dir>', 'Output directory', './data')
+    .option(
+      '--videos <mode>',
+      'Download videos: true, false, or merged (yt-dlp)',
+      'true',
+    )
+    .option(
+      '--previews <types>',
+      'Download only previews: video,audio (comma-separated)',
+    )
+    .option('--audio', 'Download standalone audio tracks', false)
+    .option('--max-posts <n>', 'Maximum number of posts to download', Number.parseInt)
+    .option(
+      '--cursor <cursor>',
+      'Resume from pagination cursor (see output of previous run)',
+    )
+    .option(
+      '--order-posts <order>',
+      'Post ordering: asc, desc, id, id_asc, id_desc',
+    )
+    .option(
+      '--order-files <order>',
+      'File ordering: r, d (reverse), or empty for natural order',
+    )
+    .option(
+      '--static-videos',
+      'Download static video cover images instead of actual videos',
+      false,
+    )
+    .option(
+      '--no-static-videos',
+      'Do not force static video covers (download real videos)',
+    )
+    .option('--api <backend>', 'API backend: rest (default) or graphql', 'rest')
+    .option('-v, --verbose', 'Verbose debug output', false)
+    .option(
+      '--include <list>',
+      'For user: comma-separated sub-extractors (posts,reels,tagged,stories,highlights,info,avatar)',
+      'posts',
+    )
+    .option(
+      '--split',
+      'For stories: split each frame into a separate post',
+      false,
+    )
+    .option(
+      '-i, --info',
+      'Print structured post info to terminal (no download)',
+      false,
+    )
+}
+
+/** Build config from parsed options */
+
+function buildConfig(opts: GlobalOptions): ConfigManager {
+  const config = new ConfigManager()
+
+  const ig: Record<string, unknown> = {}
+
+  if (opts.videos)
+    ig.videos = opts.videos
+  if (opts.previews)
+    ig.previews = opts.previews.split(',')
+  if (opts.audio)
+    ig.audio = true
+  if (opts.maxPosts)
+    ig['max-posts'] = opts.maxPosts
+  if (opts.cursor)
+    ig.cursor = opts.cursor
+  if (opts.orderPosts)
+    ig['order-posts'] = opts.orderPosts
+  if (opts.orderFiles)
+    ig['order-files'] = opts.orderFiles
+  if (opts.staticVideos)
+    ig['static-videos'] = true
+  if (opts.api)
+    ig.api = opts.api
+  if (opts.include)
+    ig.include = opts.include
+  if (opts.split)
+    ig.split = true
+
+  if (Object.keys(ig).length > 0) {
+    config.set('extractor.instagram', ig)
+  }
+
+  return config
+}
+
+/** Auto-detect the right extractor for a URL */
+
+function resolveExtractor(url: string): {
+  new (opts: any): any
+  readonly pattern: RegExp
+} {
+  for (const Cls of [
+    InstagramPostExtractor,
+    InstagramStoriesExtractor,
+    InstagramHighlightsExtractor,
+    InstagramTagExtractor,
+    InstagramSavedExtractor,
+    InstagramUserExtractor,
+  ]) {
+    if (Cls.pattern.test(url)) {
+      return Cls
+    }
+  }
+
+  throw new Error(
+    `No extractor matched URL: ${url}. `
+    + 'Supported: /p/, /reel/, /{user}/, /stories/, /highlights/, /explore/tags/, /saved/',
+  )
+}
+
+/** Run an extractor */
+
+async function runExtractor(
+  url: string,
+  extrClass: {
+    new (opts: any): any
+    readonly pattern: RegExp
+  },
+  opts: GlobalOptions,
+): Promise<void> {
+  const config = buildConfig(opts)
+  const log = createLogger(opts.verbose ?? false)
+  let http: ReturnType<typeof createHttpClient>
+  let webCsrf: string | undefined
+
+  if (opts.cookies) {
+    // Full cookie string — highest priority
+    http = createHttpClient(undefined, opts.cookies, log)
+    webCsrf = extractCsrfFromCookies(opts.cookies)
+  }
+  else if (opts.sessionid) {
+    http = createHttpClient(opts.sessionid, undefined, log)
+  }
+  else {
+    const wc = await createWebClient(log)
+    http = wc.http
+    webCsrf = wc.csrfToken
+  }
+  const storage = createStorage()
+
+  const match = extrClass.pattern.exec(url)
+  if (!match) {
+    console.error(`URL did not match expected pattern: ${url}`)
+    process.exit(1)
+  }
+
+  // eslint-disable-next-line new-cap
+  const extractor = new extrClass({
+    url,
+    match,
+    config,
+    http,
+    storage,
+    log,
+    sessionId: opts.sessionid,
+    csrfToken: webCsrf,
+  })
+
+  if (opts.info) {
+    const job = new PrintJob(
+      extractor as import('../core/extractor').Extractor,
+    )
+    const start = Date.now()
+    try {
+      const status = await job.run()
+      const elapsed = ((Date.now() - start) / 1000).toFixed(1)
+      if (status !== 0) {
+        log.warn(`Finished with status ${status} in ${elapsed}s`)
+      }
+    }
+    catch (err) {
+      log.error(String(err))
+      process.exit(1)
+    }
+    return
+  }
+
+  const job = new DownloadJob(
+    extractor as import('../core/extractor').Extractor,
+  )
+  job.basePath = opts.output ?? './data'
+
+  const start = Date.now()
+  try {
+    const status = await job.run()
+    const elapsed = ((Date.now() - start) / 1000).toFixed(1)
+
+    if (status === 0) {
+      log.info(`Done in ${elapsed}s`)
+    }
+    else {
+      log.warn(`Finished with status ${status} in ${elapsed}s`)
+    }
+  }
+  catch (err) {
+    log.error(String(err))
+    process.exit(1)
+  }
+}
+
+/** Program */
+
+const program = new Command()
+
+program
+  .name('gdl-instagram')
+  .description(
+    'Download images and videos from Instagram.\n\n'
+    + 'Uses gallery-dl\'s extraction pipeline — supports posts, reels,\n'
+    + 'stories, highlights, tagged posts, saved collections, and more.\n\n'
+    + 'Requires a sessionid cookie exported from your browser.\n'
+    + 'Set via --sessionid or INSTAGRAM_SESSIONID environment variable.',
+  )
+  .version('0.1.0')
+
+/** Default command — auto-detect extractor from URL */
+
+const dlCmd = program
+  .command('dl', { isDefault: true })
+  .argument('[url]', 'Instagram URL to download (auto-detects type)')
+  .description(
+    'Download media from an Instagram URL (auto-detects post/user/stories/…)\n\n'
+    + 'Examples:\n'
+    + '  gdl-instagram https://www.instagram.com/p/CxAbCdEfGh/\n'
+    + '  gdl-instagram https://www.instagram.com/username/ --include=posts,reels\n'
+    + '  gdl-instagram https://www.instagram.com/stories/username/',
+  )
+  .action(async (url: string | undefined, opts: GlobalOptions) => {
+    if (!url) {
+      program.help()
+      return
+    }
+    const ExtrClass = resolveExtractor(url)
+    await runExtractor(url, ExtrClass, opts)
+  })
+addSharedOptions(dlCmd)
+
+/** tag subcommand */
+
+const tag = program
+  .command('tag <hashtag>')
+  .description(
+    'Download posts from an Instagram hashtag\n\n'
+    + 'Examples:\n'
+    + '  gdl-instagram tag cats\n'
+    + '  gdl-instagram tag https://www.instagram.com/explore/tags/cats/',
+  )
+  .action(async (hashtag: string, opts: GlobalOptions) => {
+    // Accept both raw tags and full URLs
+    const url = hashtag.startsWith('http')
+      ? hashtag
+      : `https://www.instagram.com/explore/tags/${hashtag}/`
+    await runExtractor(url, InstagramTagExtractor, opts)
+  })
+addSharedOptions(tag)
+
+/** saved subcommand */
+
+const saved = program
+  .command('saved')
+  .description(
+    'Download your saved (bookmarked) posts\n\n'
+    + 'Requires authentication via --sessionid.\n\n'
+    + 'Examples:\n'
+    + '  gdl-instagram saved --sessionid=abc123',
+  )
+  .action(async (opts: GlobalOptions) => {
+    const url = 'https://www.instagram.com/me/saved/'
+    await runExtractor(url, InstagramSavedExtractor, opts)
+  })
+addSharedOptions(saved)
+
+/** parse */
+
+program.parse()