@chilfish/gallery-dl-instagram 0.1.0

package/instagram/types.ts

@@ -0,0 +1,244 @@
+/**
+ * Instagram API & parsed-post type definitions.
+ *
+ * These mirror the actual JSON shapes returned by Instagram's internal API
+ * (``/api/v1/…``).
+ */
+
+/** API response types */
+
+export interface InstagramUser {
+  pk: string
+  id?: string
+  username: string
+  full_name: string
+  is_private?: boolean
+  profile_pic_url?: string
+  profile_pic_url_hd?: string
+  hd_profile_pic_url_info?: ImageCandidate
+  hd_profile_pic_versions?: ImageCandidate[]
+  profile_pic_id?: string
+  edge_owner_to_timeline_media?: { count: number }
+  edge_felix_video_timeline?: { count: number }
+  edge_saved_media?: { count: number }
+  edge_mutual_followed_by?: { count: number }
+  edge_follow?: { count: number }
+  edge_followed_by?: { count: number }
+  edge_media_collections?: { count: number }
+  followed_by_viewer?: boolean
+}
+
+export interface ImageCandidate {
+  url: string
+  width: number
+  height: number
+}
+
+export interface VideoVersion {
+  url: string
+  width: number
+  height: number
+  type: number
+}
+
+export interface InstagramLocation {
+  pk: string
+  short_name: string
+}
+
+export interface Caption {
+  text: string
+}
+
+export interface UserTag {
+  user: InstagramUser
+}
+
+export interface ReelMention {
+  user: InstagramUser
+}
+
+export interface BloksSticker {
+  bloks_sticker: {
+    bloks_sticker_type: string
+    sticker_data: {
+      ig_mention: {
+        account_id: string
+        username: string
+        full_name: string
+      }
+    }
+  }
+}
+
+export interface MusicSticker {
+  music_asset_info?: MusicAssetInfo
+  music_consumption_info?: MusicConsumptionInfo
+}
+
+export interface MusicAssetInfo {
+  id: string
+  title?: string
+  display_artist?: string
+  ig_artist?: string
+  duration_in_ms?: number
+  highlight_start_times_in_ms?: number[]
+  progressive_download_url: string
+  cover_artwork_uri?: string
+}
+
+export interface MusicConsumptionInfo {
+  display_artist?: string
+  ig_artist?: string
+}
+
+export interface MusicMetadata {
+  music_info?: MusicAssetInfo
+}
+
+export interface InstagramPost {
+  pk: string
+  id?: string
+  code: string
+  caption: Caption | null
+  taken_at: number
+  created_at?: number
+  like_count?: number
+  has_liked?: boolean
+  user: InstagramUser
+  carousel_media?: InstagramCarouselItem[]
+  image_versions2: { candidates: ImageCandidate[] }
+  video_versions?: VideoVersion[]
+  video_dash_manifest?: string
+  original_width?: number
+  original_height?: number
+  media_type?: number
+  original_media_type?: number
+  location?: InstagramLocation
+  coauthor_producers?: InstagramUser[]
+  usertags?: { in: UserTag[] }
+  reel_mentions?: ReelMention[]
+  story_bloks_stickers?: BloksSticker[]
+  story_music_stickers?: MusicSticker[]
+  music_metadata?: MusicMetadata
+  expiring_at?: number
+  seen?: number
+  items?: InstagramCarouselItem[]
+  timeline_pinned_user_ids?: string[]
+  clips_tab_pinned_user_ids?: string[]
+  subscription_media_visibility?: string
+  audience?: string
+  title?: string
+  pins?: unknown[]
+}
+
+export interface InstagramCarouselItem {
+  pk: string
+  id?: string
+  code?: string
+  taken_at?: number
+  image_versions2: { candidates: ImageCandidate[] }
+  video_versions?: VideoVersion[]
+  video_dash_manifest?: string
+  original_width?: number
+  original_height?: number
+  media_type?: number
+  original_media_type?: number
+  owner?: InstagramUser
+  reshared_story_media_author?: InstagramUser
+  expiring_at?: number
+  subscription_media_visibility?: string
+  audience?: string
+  story_music_stickers?: MusicSticker[]
+  usertags?: { in: UserTag[] }
+  reel_mentions?: ReelMention[]
+  story_bloks_stickers?: BloksSticker[]
+}
+
+/** Parsed post (normalized output) */
+
+export interface ParsedPost {
+  post_id: string
+  post_shortcode: string
+  post_url: string
+  owner_id: string
+  username: string
+  fullname: string
+  post_date: string
+  date: string
+  description: string
+  tags?: string[]
+  location_id?: string
+  location_slug?: string
+  location_url?: string
+  likes: number
+  liked: boolean
+  pinned: string[]
+  coauthors?: Coauthor[]
+  sidecar_media_id?: string
+  sidecar_shortcode?: string
+  type: 'post' | 'reel' | 'story' | 'highlight'
+  count: number
+  _files: ParsedMedia[]
+  user?: InstagramUser
+  expires?: string
+  highlight_title?: string
+  tagged_owner_id?: string
+  tagged_username?: string
+  tagged_full_name?: string
+  subscription?: string
+  /** For graphql: */
+  typename?: string
+}
+
+export interface ParsedMedia {
+  num: number
+  date: string
+  media_id: string
+  shortcode: string
+  display_url: string
+  video_url: string | null
+  width: number
+  width_original: number
+  height: number
+  height_original: number
+  tagged_users: TaggedUser[]
+  owner?: InstagramUser
+  author?: InstagramUser
+  expires?: string
+  subscription?: string
+  audience?: string
+  audio_url?: string
+  audio_user?: string
+  audio_title?: string
+  audio_artist?: string
+  audio_duration?: number
+  audio_timestamps?: number[]
+  _ytdl_manifest_data?: string
+  sidecar_media_id?: string
+  sidecar_shortcode?: string
+}
+
+export interface Coauthor {
+  id: string
+  username: string
+  full_name?: string
+}
+
+export interface TaggedUser {
+  id: string
+  username: string
+  full_name: string
+}
+
+/** Parser config */
+
+export interface ParserConfig {
+  root: string
+  findTags: (text: string) => string[]
+  parseTimestamp: (ts: number | null | undefined) => string
+  staticVideo: boolean
+  warnVideo: boolean
+  warnImage: number
+  videosDash: boolean
+}

package/message.ts

@@ -0,0 +1,31 @@
+/**
+ * Message constructors and types.
+ *
+ * Simple factory functions so consumers never write raw message objects.
+ */
+
+import type {
+  DirectoryMsg,
+  ExtractorClass,
+  Metadata,
+  QueueMsg,
+  UrlMsg,
+} from './types'
+
+// Re-export types for convenience
+export type { Message, MessageIter } from './types'
+
+export function directory(metadata: Metadata = {}): DirectoryMsg {
+  return { type: 'directory', metadata }
+}
+
+export function url(u: string, metadata: Metadata = {}): UrlMsg {
+  return { type: 'url', url: u, metadata }
+}
+
+export function queue(
+  u: string,
+  metadata: Metadata & { _extractor?: ExtractorClass } = {},
+): QueueMsg {
+  return { type: 'queue', url: u, metadata }
+}

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@chilfish/gallery-dl-instagram",
+  "type": "module",
+  "version": "0.1.0",
+  "description": "Instagram extraction pipeline — platform-agnostic SDK + CLI",
+  "license": "GPL-2.0-only",
+  "keywords": [
+    "instagram",
+    "scraper",
+    "downloader",
+    "gallery-dl",
+    "cli"
+  ],
+  "exports": {
+    ".": "./dist/index.mjs",
+    "./cli": "./dist/cli/index.mjs",
+    "./sdk": "./dist/sdk.mjs",
+    "./package.json": "./package.json"
+  },
+  "types": "./dist/index.d.mts",
+  "bin": {
+    "gallery-dl-instagram": "./dist/cli/index.mjs"
+  },
+  "files": [
+    "!*.log",
+    "!node_modules",
+    "cli/",
+    "config.ts",
+    "core/",
+    "dist/",
+    "index.ts",
+    "instagram/",
+    "message.ts",
+    "types.ts",
+    "utils/"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "typecheck": "tsc --noEmit -p tsconfig.json",
+    "cli": "bun cli/index.ts",
+    "lint": "eslint . --fix",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "test:unit": "vitest run tests/unit",
+    "test:integration": "vitest run tests/integration"
+  },
+  "dependencies": {
+    "axios": "^1.16.1"
+  },
+  "devDependencies": {
+    "@antfu/eslint-config": "^9.0.0",
+    "@types/node": "^25.9.1",
+    "commander": "^14.0.3",
+    "dotenv": "^17.4.2",
+    "eslint": "^10.4.0",
+    "lefthook": "^2.1.8",
+    "tsdown": "^0.22.0",
+    "typescript": "^6.0.3",
+    "vitest": "^4.1.7"
+  },
+  "inlinedDependencies": {
+    "commander": "14.0.3"
+  }
+}

package/types.ts

@@ -0,0 +1,115 @@
+/**
+ * Shared type definitions for the gallery-dl TypeScript port.
+ *
+ * Message types form a discriminated union — the `type` tag determines
+ * which handler a Job should invoke.
+ */
+
+/** HTTP abstraction */
+
+export interface RequestConfig {
+  headers?: Record<string, string>
+  params?: Record<string, string | number | null | undefined>
+  method?: string
+  data?: unknown
+  timeout?: number
+  signal?: AbortSignal
+  /** WithCredentials / CORS cookie passthrough for browsers */
+  withCredentials?: boolean
+  /** For binary downloads — 'arraybuffer' returns raw bytes */
+  responseType?: 'arraybuffer' | 'text' | 'json'
+}
+
+export interface HttpResponse<T = unknown> {
+  status: number
+  data: T
+  headers: Record<string, string>
+  /** Final URL after redirects */
+  url: string
+  request?: RequestConfig
+}
+
+export interface HttpClient {
+  request: <T = unknown>(config: {
+    url: string
+    method?: string
+    headers?: Record<string, string>
+    params?: Record<string, string | number | null | undefined>
+    data?: unknown
+    signal?: AbortSignal
+    timeout?: number
+    responseType?: 'arraybuffer' | 'text' | 'json'
+  }) => Promise<HttpResponse<T>>
+}
+
+/** Storage abstraction */
+
+export interface Storage {
+  exists: (path: string) => Promise<boolean>
+  write: (path: string, data: Uint8Array | string) => Promise<void>
+  mkdir: (path: string) => Promise<void>
+}
+
+export type ConfigValue
+  = | string
+    | number
+    | boolean
+    | null
+    | ConfigValue[]
+    | { [key: string]: ConfigValue }
+
+export interface Config { [key: string]: ConfigValue }
+
+/** Metadata & messages */
+
+/**
+ * Flat string-keyed metadata dictionary.
+ * In gallery-dl every kwdict is a plain `{string → value}` map.
+ */
+export type Metadata = Record<string, unknown>
+
+export interface DirectoryMsg {
+  readonly type: 'directory'
+  readonly metadata: Metadata
+}
+
+export interface UrlMsg {
+  readonly type: 'url'
+  readonly url: string
+  readonly metadata: Metadata
+}
+
+export interface QueueMsg {
+  readonly type: 'queue'
+  readonly url: string
+  readonly metadata: Metadata & {
+    readonly _extractor?: ExtractorClass
+  }
+}
+
+export type Message = DirectoryMsg | UrlMsg | QueueMsg
+
+/**
+ * Async generator that yields Message values.
+ */
+export type MessageIter = AsyncGenerator<Message, void, unknown>
+
+/** Extractor class reference (for Queue dispatch) */
+
+/**
+ * Minimal shape that every Extractor class must expose so the Dispatch
+ * logic can re-instantiate from a URL.
+ */
+export interface ExtractorClass {
+  pattern: RegExp
+  subcategory: string
+}
+
+// Pre-declare the Extractor interface to avoid circular refs
+export interface Extractor {
+  readonly category: string
+  readonly subcategory: string
+  readonly root: string
+  initialize: () => Promise<void>
+  [Symbol.asyncIterator]: () => MessageIter
+}

package/utils/id-codec.ts

@@ -0,0 +1,39 @@
+/**
+ * Instagram-style Base64-variant ID ↔ shortcode conversion.
+ */
+
+const ALPHABET
+  = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'
+
+/** Pre-built index for O(1) character lookup during decode. */
+const CHAR_INDEX: Record<string, number> = {}
+for (let i = 0; i < ALPHABET.length; i++) {
+  CHAR_INDEX[ALPHABET[i]!] = i
+}
+
+const BASE = BigInt(ALPHABET.length)
+
+/**
+ * Decode an Instagram shortcode into its numeric post ID.
+ */
+export function idFromShortcode(shortcode: string): string {
+  let num = 0n
+  for (const ch of shortcode) {
+    num = num * BASE + BigInt(CHAR_INDEX[ch] ?? 0)
+  }
+  return num.toString()
+}
+
+/**
+ * Encode a numeric post ID into an Instagram shortcode.
+ */
+export function shortcodeFromId(postId: string | number): string {
+  let num = BigInt(postId)
+  const chars: string[] = []
+  while (num > 0n) {
+    const remainder = Number(num % BASE)
+    chars.push(ALPHABET[remainder]!)
+    num = num / BASE
+  }
+  return chars.reverse().join('')
+}

package/utils/text.ts

@@ -0,0 +1,178 @@
+/**
+ * Text utilities ported from gallery-dl's ``text`` module.
+ *
+ * All functions are pure and environment-agnostic.
+ */
+
+/** String extraction */
+
+/**
+ * Extract the substring between ``begin`` and ``end`` from ``txt``.
+ * Returns the substring or ``null`` if either delimiter is missing.
+ */
+export function extract(
+  txt: string,
+  begin: string,
+  end: string,
+): string | null {
+  const first = txt.indexOf(begin)
+  if (first < 0)
+    return null
+  const start = first + begin.length
+  const last = txt.indexOf(end, start)
+  if (last < 0)
+    return null
+  return txt.slice(start, last)
+}
+
+/**
+ * Shorthand: same as ``extract`` but returns ``default_`` on failure.
+ * Mirrors the Python ``extr()`` function.
+ */
+export function extr(
+  txt: string,
+  begin: string,
+  end: string,
+  default_ = '',
+): string {
+  return extract(txt, begin, end) ?? default_
+}
+
+/** Unicode / HTML */
+
+/**
+ * Decode ``\\uXXXX`` escape sequences in a string.
+ */
+export function parseUnicodeEscapes(text: string): string {
+  if (!text.includes('\\u'))
+    return text
+  return text.replace(/\\u([0-9a-fA-F]{4})/g, (_m, hex) =>
+    String.fromCharCode(Number.parseInt(hex, 16)))
+}
+
+/**
+ * HTML entity decode.
+ *
+ * In Node.js we could use a DOM parser, but since this library is
+ * environment-agnostic we ship a minimal covering the common cases.
+ */
+const HTML_ENTITIES: Record<string, string> = {
+  'amp': '&',
+  'lt': '<',
+  'gt': '>',
+  'quot': '"',
+  'apos': '\'',
+  'nbsp': '\u00A0',
+  '#x27': '\'',
+  '#x2F': '/',
+  '#39': '\'',
+  '#47': '/',
+}
+
+const RE_ENTITY = /&([^;]+);/g
+
+export function unescape(text: string): string {
+  return text.replace(RE_ENTITY, (m, name) => {
+    const ch = HTML_ENTITIES[name]
+    if (ch !== undefined)
+      return ch
+    // Numeric entities like &#123;
+    if (name.startsWith('#')) {
+      const cp = name[1] === 'x' || name[1] === 'X'
+        ? Number.parseInt(name.slice(2), 16)
+        : Number.parseInt(name.slice(1), 10)
+      if (Number.isSafeInteger(cp))
+        return String.fromCodePoint(cp)
+    }
+    return m // leave unrecognized as-is
+  })
+}
+
+/** URL helpers */
+
+/**
+ * URL-decode a string.
+ */
+export function unquote(text: string): string {
+  try {
+    return decodeURIComponent(text)
+  }
+  catch {
+    // Best-effort: replace invalid sequences
+    return text.replace(/%[0-9a-f]{2}/gi, (m) => {
+      try {
+        return decodeURIComponent(m)
+      }
+      catch {
+        return m
+      }
+    })
+  }
+}
+
+/**
+ * Ensure a URL starts with ``https://`` (or ``http://``).
+ */
+export function ensureHttpScheme(url: string, scheme = 'https://'): string {
+  if (!url)
+    return url
+  if (url.startsWith('https://') || url.startsWith('http://'))
+    return url
+  return scheme + url.replace(/^[/:]+/, '')
+}
+
+/**
+ * Extract filename + extension from a URL and write into ``meta``.
+ */
+export function nameExtFromURL(
+  url: string,
+  meta: Record<string, unknown>,
+): void {
+  const filename = filenameFromURL(url)
+  const dot = filename.lastIndexOf('.')
+  if (dot > 0 && filename.length - dot - 1 <= 16) {
+    meta.filename = unquote(filename.slice(0, dot))
+    meta.extension = unquote(filename.slice(dot + 1)).toLowerCase()
+  }
+  else {
+    meta.filename = unquote(filename)
+    meta.extension = ''
+  }
+}
+
+/**
+ * Extract the file-name portion of a URL (before query string).
+ */
+function filenameFromURL(url: string): string {
+  try {
+    return url.split('?')[0]!.split('/').pop() ?? ''
+  }
+  catch {
+    return ''
+  }
+}
+
+/**
+ * Parse an integer from a possibly-null value. Returns ``default_`` on failure.
+ */
+export function parseInt(
+  value: string | number | null | undefined,
+  default_: number = 0,
+): number {
+  if (value == null)
+    return default_
+  const n = typeof value === 'number' ? value : Number.parseInt(String(value), 10)
+  return Number.isFinite(n) ? n : default_
+}
+
+// Regex helper: compile a pattern and call .findall()
+export function tagRe(pattern: string): (text: string) => string[] {
+  const re = new RegExp(pattern, 'g')
+  return (text: string) => {
+    const matches = text.match(re)
+    return matches ? [...new Set(matches)] : []
+  }
+}
+
+/** Pre-configured hashtag regex. */
+export const findTags = tagRe('#\\w+')