@steipete/summarize-core 0.7.0

package/dist/types/content/transcript/providers/youtube/api.d.ts

@@ -0,0 +1,26 @@
+export interface YoutubeTranscriptConfig {
+    apiKey: string;
+    context: Record<string, unknown>;
+    params: string;
+    visitorData?: string | null;
+    clientName?: string | null;
+    clientVersion?: string | null;
+    pageCl?: number | null;
+    pageLabel?: string | null;
+}
+export declare const extractYoutubeiTranscriptConfig: (html: string) => YoutubeTranscriptConfig | null;
+export declare const fetchTranscriptFromTranscriptEndpoint: (fetchImpl: typeof fetch, { config, originalUrl, }: {
+    config: YoutubeTranscriptConfig;
+    originalUrl: string;
+}) => Promise<string | null>;
+export declare const extractTranscriptFromTranscriptEndpoint: (data: unknown) => string | null;
+export declare const extractYoutubeiBootstrap: (html: string) => {
+    apiKey: string | null;
+    context: Record<string, unknown>;
+    clientVersion: string | null;
+    clientName: string | null;
+    visitorData: string | null;
+    pageCl: number | null;
+    pageLabel: string | null;
+    xsrfToken: string | null;
+} | null;

package/dist/types/content/transcript/providers/youtube/apify.d.ts

@@ -0,0 +1 @@
+export declare const fetchTranscriptWithApify: (fetchImpl: typeof fetch, apifyApiToken: string | null, url: string) => Promise<string | null>;

package/dist/types/content/transcript/providers/youtube/captions.d.ts

@@ -0,0 +1,7 @@
+interface YoutubeTranscriptContext {
+    html: string;
+    originalUrl: string;
+    videoId: string;
+}
+export declare const fetchTranscriptFromCaptionTracks: (fetchImpl: typeof fetch, { html, originalUrl, videoId }: YoutubeTranscriptContext) => Promise<string | null>;
+export {};

package/dist/types/content/transcript/providers/youtube/yt-dlp.d.ts

@@ -0,0 +1,17 @@
+import { type TranscriptionProvider } from '../../../../transcription/whisper.js';
+import type { LinkPreviewProgressEvent } from '../../../link-preview/deps.js';
+type YtDlpTranscriptResult = {
+    text: string | null;
+    provider: TranscriptionProvider | null;
+    error: Error | null;
+    notes: string[];
+};
+type YtDlpRequest = {
+    ytDlpPath: string | null;
+    openaiApiKey: string | null;
+    falApiKey: string | null;
+    url: string;
+    onProgress?: ((event: LinkPreviewProgressEvent) => void) | null;
+};
+export declare const fetchTranscriptWithYtDlp: ({ ytDlpPath, openaiApiKey, falApiKey, url, onProgress, }: YtDlpRequest) => Promise<YtDlpTranscriptResult>;
+export {};

package/dist/types/content/transcript/providers/youtube.d.ts

@@ -0,0 +1,3 @@
+import type { ProviderContext, ProviderFetchOptions, ProviderResult } from '../types.js';
+export declare const canHandle: ({ url }: ProviderContext) => boolean;
+export declare const fetchTranscript: (context: ProviderContext, options: ProviderFetchOptions) => Promise<ProviderResult>;

package/dist/types/content/transcript/types.d.ts

@@ -0,0 +1,30 @@
+import type { YoutubeTranscriptMode } from '../link-preview/content/types.js';
+import type { LinkPreviewProgressEvent, ScrapeWithFirecrawl } from '../link-preview/deps.js';
+import type { TranscriptResolution, TranscriptSource } from '../link-preview/types.js';
+export type TranscriptService = 'youtube' | 'podcast' | 'generic';
+export interface ProviderContext {
+    url: string;
+    html: string | null;
+    resourceKey: string | null;
+}
+export interface ProviderFetchOptions {
+    fetch: typeof fetch;
+    scrapeWithFirecrawl?: ScrapeWithFirecrawl | null;
+    apifyApiToken: string | null;
+    youtubeTranscriptMode: YoutubeTranscriptMode;
+    ytDlpPath: string | null;
+    falApiKey: string | null;
+    openaiApiKey: string | null;
+    onProgress?: ((event: LinkPreviewProgressEvent) => void) | null;
+}
+export interface ProviderResult extends TranscriptResolution {
+    metadata?: Record<string, unknown>;
+    attemptedProviders: TranscriptSource[];
+    notes?: string | null;
+}
+export interface ProviderModule {
+    id: TranscriptService;
+    canHandle(context: ProviderContext): boolean;
+    fetchTranscript(context: ProviderContext, options: ProviderFetchOptions): Promise<ProviderResult>;
+}
+export type { TranscriptSource } from '../link-preview/types.js';

package/dist/types/content/transcript/utils.d.ts

@@ -0,0 +1,8 @@
+export declare function isRecord(value: unknown): value is Record<string, unknown>;
+export declare const isYouTubeUrl: (rawUrl: string) => boolean;
+export declare function isYouTubeVideoUrl(rawUrl: string): boolean;
+export declare function extractYouTubeVideoId(rawUrl: string): string | null;
+export declare function extractEmbeddedYouTubeUrlFromHtml(html: string, maxTextChars?: number, maxReadabilityChars?: number): Promise<string | null>;
+export declare function sanitizeYoutubeJsonResponse(input: string): string;
+export declare function decodeHtmlEntities(input: string): string;
+export declare function extractYoutubeBootstrapConfig(html: string): Record<string, unknown> | null;

package/dist/types/index.d.ts

@@ -0,0 +1,4 @@
+export * from './content/index.js';
+export * from './prompts/index.js';
+export type { SummaryLength } from './shared/contracts.js';
+export { SUMMARY_LENGTHS } from './shared/contracts.js';

package/dist/types/language.d.ts

@@ -0,0 +1,25 @@
+export type OutputLanguage = {
+    kind: 'auto';
+} | {
+    kind: 'fixed';
+    /**
+     * BCP-47-ish language tag (e.g. "en", "de", "en-US").
+     *
+     * Note: we keep this mostly user-provided; the model does the heavy lifting.
+     */
+    tag: string;
+    /**
+     * Human-friendly label for prompts (e.g. "English", "German").
+     */
+    label: string;
+};
+export declare function parseOutputLanguage(raw: string): OutputLanguage;
+export declare function resolveOutputLanguage(raw: string | null | undefined): OutputLanguage;
+export declare function formatOutputLanguageInstruction(language: OutputLanguage): string;
+export declare function formatOutputLanguageForJson(language: OutputLanguage): {
+    mode: 'auto';
+} | {
+    mode: 'fixed';
+    tag: string;
+    label: string;
+};

package/dist/types/prompts/cli.d.ts

@@ -0,0 +1,10 @@
+import type { OutputLanguage } from '../language.js';
+import type { SummaryLengthTarget } from './link-summary.js';
+export declare function buildPathSummaryPrompt({ kindLabel, filePath, filename, mediaType, outputLanguage, summaryLength, }: {
+    kindLabel: 'file' | 'image';
+    filePath: string;
+    filename: string | null;
+    mediaType: string | null;
+    summaryLength: SummaryLengthTarget;
+    outputLanguage?: OutputLanguage | null;
+}): string;

package/dist/types/prompts/file.d.ts

@@ -0,0 +1,17 @@
+import type { OutputLanguage } from '../language.js';
+import type { SummaryLengthTarget } from './link-summary.js';
+export declare function buildFileSummaryPrompt({ filename, mediaType, outputLanguage, summaryLength, contentLength, }: {
+    filename: string | null;
+    mediaType: string | null;
+    summaryLength: SummaryLengthTarget;
+    contentLength?: number | null;
+    outputLanguage?: OutputLanguage | null;
+}): string;
+export declare function buildFileTextSummaryPrompt({ filename, originalMediaType, contentMediaType, outputLanguage, summaryLength, contentLength, }: {
+    filename: string | null;
+    originalMediaType: string | null;
+    contentMediaType: string;
+    summaryLength: SummaryLengthTarget;
+    contentLength: number;
+    outputLanguage?: OutputLanguage | null;
+}): string;

package/dist/types/prompts/index.d.ts

@@ -0,0 +1,4 @@
+export type { SummaryLength } from '../shared/contracts.js';
+export { buildPathSummaryPrompt } from './cli.js';
+export { buildFileSummaryPrompt, buildFileTextSummaryPrompt } from './file.js';
+export { buildLinkSummaryPrompt, estimateMaxCompletionTokensForCharacters, pickSummaryLengthForCharacters, type ShareContextEntry, SUMMARY_LENGTH_TO_TOKENS, type SummaryLengthTarget, } from './link-summary.js';

package/dist/types/prompts/link-summary.d.ts

@@ -0,0 +1,29 @@
+import type { OutputLanguage } from '../language.js';
+import type { SummaryLength } from '../shared/contracts.js';
+export declare const SUMMARY_LENGTH_TO_TOKENS: Record<SummaryLength, number>;
+export type SummaryLengthTarget = SummaryLength | {
+    maxCharacters: number;
+};
+export declare function pickSummaryLengthForCharacters(maxCharacters: number): SummaryLength;
+export declare function estimateMaxCompletionTokensForCharacters(maxCharacters: number): number;
+export type ShareContextEntry = {
+    author: string;
+    handle?: string | null;
+    text: string;
+    likeCount?: number | null;
+    reshareCount?: number | null;
+    replyCount?: number | null;
+    timestamp?: string | null;
+};
+export declare function buildLinkSummaryPrompt({ url, title, siteName, description, content, truncated, hasTranscript, outputLanguage, summaryLength, shares, }: {
+    url: string;
+    title: string | null;
+    siteName: string | null;
+    description: string | null;
+    content: string;
+    truncated: boolean;
+    hasTranscript: boolean;
+    summaryLength: SummaryLengthTarget;
+    outputLanguage?: OutputLanguage | null;
+    shares: ShareContextEntry[];
+}): string;

package/dist/types/shared/contracts.d.ts

@@ -0,0 +1,2 @@
+export declare const SUMMARY_LENGTHS: readonly ["short", "medium", "long", "xl", "xxl"];
+export type SummaryLength = (typeof SUMMARY_LENGTHS)[number];

package/dist/types/transcription/whisper/constants.d.ts

@@ -0,0 +1,7 @@
+export declare const TRANSCRIPTION_TIMEOUT_MS = 600000;
+export declare const MAX_ERROR_DETAIL_CHARS = 200;
+export declare const MAX_OPENAI_UPLOAD_BYTES: number;
+export declare const DEFAULT_SEGMENT_SECONDS = 600;
+export declare const DISABLE_LOCAL_WHISPER_CPP_ENV = "SUMMARIZE_DISABLE_LOCAL_WHISPER_CPP";
+export declare const WHISPER_CPP_MODEL_PATH_ENV = "SUMMARIZE_WHISPER_CPP_MODEL_PATH";
+export declare const WHISPER_CPP_BINARY_ENV = "SUMMARIZE_WHISPER_CPP_BINARY";

package/dist/types/transcription/whisper/core.d.ts

@@ -0,0 +1,20 @@
+import type { WhisperProgressEvent, WhisperTranscriptionResult } from './types.js';
+export declare function transcribeMediaWithWhisper({ bytes, mediaType, filename, openaiApiKey, falApiKey, totalDurationSeconds, onProgress, }: {
+    bytes: Uint8Array;
+    mediaType: string;
+    filename: string | null;
+    openaiApiKey: string | null;
+    falApiKey: string | null;
+    totalDurationSeconds?: number | null;
+    onProgress?: ((event: WhisperProgressEvent) => void) | null;
+}): Promise<WhisperTranscriptionResult>;
+export declare function transcribeMediaFileWithWhisper({ filePath, mediaType, filename, openaiApiKey, falApiKey, segmentSeconds, totalDurationSeconds, onProgress, }: {
+    filePath: string;
+    mediaType: string;
+    filename: string | null;
+    openaiApiKey: string | null;
+    falApiKey: string | null;
+    segmentSeconds?: number;
+    totalDurationSeconds?: number | null;
+    onProgress?: ((event: WhisperProgressEvent) => void) | null;
+}): Promise<WhisperTranscriptionResult>;

package/dist/types/transcription/whisper/fal.d.ts

@@ -0,0 +1 @@
+export declare function transcribeWithFal(bytes: Uint8Array, mediaType: string, apiKey: string): Promise<string | null>;

package/dist/types/transcription/whisper/ffmpeg.d.ts

@@ -0,0 +1,16 @@
+export declare function isFfmpegAvailable(): Promise<boolean>;
+export declare function probeMediaDurationSecondsWithFfprobe(filePath: string): Promise<number | null>;
+export declare function runFfmpegSegment({ inputPath, outputPattern, segmentSeconds, }: {
+    inputPath: string;
+    outputPattern: string;
+    segmentSeconds: number;
+}): Promise<void>;
+export declare function runFfmpegTranscodeToMp3({ inputPath, outputPath, }: {
+    inputPath: string;
+    outputPath: string;
+}): Promise<void>;
+export declare function runFfmpegTranscodeToMp3Lenient({ inputPath, outputPath, }: {
+    inputPath: string;
+    outputPath: string;
+}): Promise<void>;
+export declare function transcodeBytesToMp3(bytes: Uint8Array): Promise<Uint8Array>;

package/dist/types/transcription/whisper/openai.d.ts

@@ -0,0 +1,2 @@
+export declare function transcribeWithOpenAi(bytes: Uint8Array, mediaType: string, filename: string | null, apiKey: string): Promise<string | null>;
+export declare function shouldRetryOpenAiViaFfmpeg(error: Error): boolean;

package/dist/types/transcription/whisper/types.d.ts

@@ -0,0 +1,17 @@
+export type TranscriptionProvider = 'openai' | 'fal' | 'whisper.cpp';
+export type WhisperTranscriptionResult = {
+    text: string | null;
+    provider: TranscriptionProvider | null;
+    error: Error | null;
+    notes: string[];
+};
+export type WhisperProgressEvent = {
+    /** 1-based segment index (only when chunked via ffmpeg). */
+    partIndex: number | null;
+    /** Total number of segments (only when chunked via ffmpeg). */
+    parts: number | null;
+    /** Best-effort processed duration of the source media. */
+    processedDurationSeconds: number | null;
+    /** Best-effort total duration of the source media. */
+    totalDurationSeconds: number | null;
+};

package/dist/types/transcription/whisper/utils.d.ts

@@ -0,0 +1,5 @@
+export declare function wrapError(prefix: string, error: unknown): Error;
+export declare function toArrayBuffer(view: Uint8Array): ArrayBuffer;
+export declare function ensureWhisperFilenameExtension(name: string, mediaType: string): string;
+export declare function readFirstBytes(filePath: string, maxBytes: number): Promise<Uint8Array>;
+export declare function formatBytes(bytes: number): string;

package/dist/types/transcription/whisper/whisper-cpp.d.ts

@@ -0,0 +1,9 @@
+import type { WhisperProgressEvent, WhisperTranscriptionResult } from './types.js';
+export declare function isWhisperCppReady(): Promise<boolean>;
+export declare function resolveWhisperCppModelNameForDisplay(): Promise<string | null>;
+export declare function transcribeWithWhisperCppFile({ filePath, mediaType, totalDurationSeconds, onProgress, }: {
+    filePath: string;
+    mediaType: string;
+    totalDurationSeconds: number | null;
+    onProgress?: ((event: WhisperProgressEvent) => void) | null;
+}): Promise<WhisperTranscriptionResult>;

package/dist/types/transcription/whisper.d.ts

@@ -0,0 +1,5 @@
+export { MAX_OPENAI_UPLOAD_BYTES } from './whisper/constants.js';
+export { transcribeMediaFileWithWhisper, transcribeMediaWithWhisper } from './whisper/core.js';
+export { isFfmpegAvailable, probeMediaDurationSecondsWithFfprobe } from './whisper/ffmpeg.js';
+export type { TranscriptionProvider, WhisperProgressEvent, WhisperTranscriptionResult, } from './whisper/types.js';
+export { isWhisperCppReady, resolveWhisperCppModelNameForDisplay, } from './whisper/whisper-cpp.js';

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@steipete/summarize-core",
+  "version": "0.7.0",
+  "description": "Summarize core library (content extraction + prompts).",
+  "type": "module",
+  "main": "./dist/esm/index.js",
+  "module": "./dist/esm/index.js",
+  "types": "./dist/types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/esm/index.js"
+    },
+    "./content": {
+      "types": "./dist/types/content/index.d.ts",
+      "import": "./dist/esm/content/index.js"
+    },
+    "./prompts": {
+      "types": "./dist/types/prompts/index.d.ts",
+      "import": "./dist/esm/prompts/index.js"
+    },
+    "./language": {
+      "types": "./dist/types/language.d.ts",
+      "import": "./dist/esm/language.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "packageManager": "pnpm@10.25.0+sha512.5e82639027af37cf832061bcc6d639c219634488e0f2baebe785028a793de7b525ffcd3f7ff574f5e9860654e098fe852ba8ac5dd5cefe1767d23a020a92f501",
+  "engines": {
+    "node": ">=22"
+  },
+  "scripts": {
+    "clean": "rm -rf dist",
+    "build": "pnpm clean && tsc -p tsconfig.build.json",
+    "typecheck": "tsc -p tsconfig.build.json --noEmit"
+  },
+  "dependencies": {
+    "@fal-ai/client": "^1.2.1",
+    "@mozilla/readability": "0.6.0",
+    "cheerio": "^1.1.2",
+    "es-toolkit": "^1.43.0",
+    "jsdom": "27.3.0",
+    "sanitize-html": "^2.17.0"
+  },
+  "devDependencies": {
+    "@types/jsdom": "^27.0.0",
+    "@types/node": "^25.0.3",
+    "@types/sanitize-html": "^2.16.0",
+    "typescript": "^5.9.3"
+  }
+}