@officexapp/vidfarm-devcli 0.21.42 → 0.21.45

package/dist/src/services/clip-curation/media-select.js

@@ -4,6 +4,138 @@
 // by whatever source we ingest, so we must pick the HIGHEST-resolution playable
 // MP4 — not just the first one. Shared by the clip-scan Lambda (infra/lambda)
 // and the in-process serve/import path (src/app.ts) so both stay in lock-step.
+// ── Provider: social-download-all-in-one (RapidAPI) ──────────────────────────
+// The single social resolver behind every URL ingest. It takes a JSON body
+// (`{url}`) and answers 200 for BOTH success and failure — a failed lookup is
+// `{"error":true,"status":404,"message":"Not found data"}` — so the HTTP status
+// alone never tells you whether there is media. Its per-platform quirks are
+// normalized here, once, rather than in each of the five call sites:
+//
+//   youtube   `medias[].ext`, quality "mp4 (1080p)", every rendition above 360p
+//             is a DASH VIDEO-ONLY stream, and 2160p/1440p mp4 are AV1
+//   tiktok    byte size arrives as `data_size`; qualities are
+//             hd_no_watermark / no_watermark / watermark (hd is usually HEVC);
+//             `duration` is in MILLISECONDS
+//   instagram quality is "WxHp" ("720x1280p"); carries `is_audio`
+//   x         one entry plus an alternate-bitrate `formats[]` (includes an m3u8)
+//
+export const DEFAULT_SOCIAL_DOWNLOAD_URL = "https://social-download-all-in-one.p.rapidapi.com/v1/social/autolink";
+export const DEFAULT_SOCIAL_DOWNLOAD_HOST = "social-download-all-in-one.p.rapidapi.com";
+/** TikTok posts cap at 10 minutes, so any larger `duration` is milliseconds. */
+const TIKTOK_MAX_DURATION_SEC = 600;
+/** Audio codecs that can appear in an MP4/WebM `codecs="…"` parameter. */
+const AUDIO_CODEC_RE = /(^|[,\s])(mp4a|opus|vorbis|ac-3|ec-3|flac|alac|mp3)/i;
+/**
+ * Does this rendition carry audio? See `DownloadMediaCandidate.hasAudioTrack`
+ * for why `undefined` (unknown) must NOT be read as "no".
+ */
+function detectMuxedAudio(raw) {
+    if (raw.type === "audio")
+        return true;
+    if (raw.is_audio === true)
+        return true;
+    const codecs = String(raw.mimeType ?? "").match(/codecs="([^"]+)"/i)?.[1];
+    // A declared codec list is authoritative: YouTube's muxed itag-18 rendition
+    // lists `avc1…, mp4a…`, its DASH renditions list the video codec alone.
+    if (codecs)
+        return AUDIO_CODEC_RE.test(codecs);
+    if (raw.audioQuality)
+        return true;
+    return undefined;
+}
+function toFiniteNumber(value) {
+    const parsed = typeof value === "number" ? value : Number(value);
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
+}
+/** Map one provider media entry onto the canonical candidate shape. */
+export function normalizeSocialDownloadMedia(entry) {
+    const raw = (entry && typeof entry === "object" ? entry : {});
+    const type = typeof raw.type === "string" ? raw.type : undefined;
+    const hasAudioTrack = detectMuxedAudio(raw);
+    return {
+        url: typeof raw.url === "string" ? raw.url : undefined,
+        quality: typeof raw.quality === "string" ? raw.quality : undefined,
+        label: typeof raw.label === "string" ? raw.label : undefined,
+        // YouTube uses `ext`; every other platform uses `extension`.
+        extension: typeof raw.extension === "string" ? raw.extension : typeof raw.ext === "string" ? raw.ext : undefined,
+        type,
+        width: toFiniteNumber(raw.width),
+        height: toFiniteNumber(raw.height),
+        size: toFiniteNumber(raw.size) ?? toFiniteNumber(raw.data_size),
+        mimeType: typeof raw.mimeType === "string" ? raw.mimeType : undefined,
+        hasAudioTrack,
+        // The provider no longer sends the old availability flags, so derive them
+        // from the declared `type` — downstream selectors still read them.
+        videoAvailable: type ? type === "video" : undefined,
+        audioAvailable: type === "audio" ? true : type === "video" ? hasAudioTrack : undefined
+    };
+}
+function normalizeLookupDuration(rawDuration, source) {
+    const parsed = typeof rawDuration === "number" ? rawDuration : Number(rawDuration);
+    if (!Number.isFinite(parsed) || parsed <= 0)
+        return null;
+    if (source === "tiktok" && parsed > TIKTOK_MAX_DURATION_SEC)
+        return Number((parsed / 1000).toFixed(3));
+    return parsed;
+}
+function trimmedOrNull(value) {
+    return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+/**
+ * Normalize a raw provider response, throwing its own `{error, status, message}`
+ * failure as an Error. Split from the fetch so tests (and any caller holding an
+ * already-parsed body) can use it directly.
+ */
+export function normalizeSocialDownloadLookup(body) {
+    const raw = (body && typeof body === "object" ? body : {});
+    if (raw.error === true || (raw.error && typeof raw.error === "string")) {
+        const message = trimmedOrNull(raw.message) ?? (typeof raw.error === "string" ? raw.error : null) ?? "lookup failed";
+        const status = raw.status ? ` (status ${String(raw.status)})` : "";
+        throw new Error(`Video download lookup failed: ${message}${status}`);
+    }
+    const source = trimmedOrNull(raw.source);
+    return {
+        url: trimmedOrNull(raw.url),
+        source,
+        title: trimmedOrNull(raw.title),
+        author: trimmedOrNull(raw.author),
+        thumbnail: trimmedOrNull(raw.thumbnail),
+        duration: normalizeLookupDuration(raw.duration, source),
+        medias: Array.isArray(raw.medias) ? raw.medias.map(normalizeSocialDownloadMedia) : []
+    };
+}
+/**
+ * One lookup call, normalized. Every URL-ingest path in the codebase (the
+ * video_download primitive, the clip-scan Lambda, the clipper preview, the
+ * in-process import, the seed scripts) goes through here so the provider's
+ * request shape and its per-platform quirks live in exactly one place.
+ */
+export async function fetchSocialDownloadLookup(input) {
+    const endpoint = (input.endpoint || "").trim() || DEFAULT_SOCIAL_DOWNLOAD_URL;
+    const host = (input.host || "").trim() || DEFAULT_SOCIAL_DOWNLOAD_HOST;
+    const response = await fetch(endpoint, {
+        method: "POST",
+        headers: {
+            "x-rapidapi-key": input.apiKey,
+            "x-rapidapi-host": host,
+            "content-type": "application/json"
+        },
+        body: JSON.stringify({ url: input.sourceUrl })
+    });
+    if (input.onApiCall) {
+        try {
+            await input.onApiCall({ sourceUrl: input.sourceUrl, host, status: response.status });
+        }
+        catch {
+            // metering is best-effort; never fail the download over it
+        }
+    }
+    if (!response.ok) {
+        const details = await response.text().catch(() => "");
+        throw new Error(`Video download lookup failed with HTTP ${response.status}${details ? `: ${details.slice(0, 300)}` : ""}`);
+    }
+    return normalizeSocialDownloadLookup(await response.json());
+}
 /** True when a candidate is a directly-downloadable MP4 (not HLS/DASH/audio). */
 function isPlayableMp4(m) {
     if (!m.url)
@@ -88,9 +220,20 @@ export function rankPlayableVideoMedias(medias) {
         .map((m, index) => ({ m, index, h: estimateMediaHeight(m) }))
         .sort((a, b) => b.h - a.h || a.index - b.index)
         .map((entry) => entry.m);
-    const mp4s = rankByHeight(list.filter(isPlayableMp4));
-    const others = rankByHeight(list.filter((m) => !isPlayableMp4(m) && isPlayableVideo(m)));
-    return [...mp4s, ...others];
+    // Silent renditions rank BELOW every muxed one regardless of resolution. A
+    // YouTube lookup offers 2160p/1440p/1080p/… as video-only DASH streams and
+    // only ~360p muxed; taking the tallest yields a clip library with no audio,
+    // which is worse than a smaller one that sounds right. A video-only rendition
+    // is still kept as a last tier — a silent clip beats a failed ingest.
+    const silent = (m) => m.hasAudioTrack === false;
+    const mp4s = list.filter(isPlayableMp4);
+    const others = list.filter((m) => !isPlayableMp4(m) && isPlayableVideo(m));
+    return [
+        ...rankByHeight(mp4s.filter((m) => !silent(m))),
+        ...rankByHeight(others.filter((m) => !silent(m))),
+        ...rankByHeight(mp4s.filter(silent)),
+        ...rankByHeight(others.filter(silent))
+    ];
 }
 /**
  * Pick the best rendition to ingest: the highest-resolution playable MP4, with a

package/experimental/google-news-to-video.md

@@ -0,0 +1,235 @@
+# Google News to Video — reusable prompt & method
+
+Turns **a recent real event → a timely short video**, using two searches instead of one.
+
+Google News is excellent at finding **stories**. It is bad at finding **footage** — it returns
+articles. So the method splits in two, and the split is the whole point:
+
+1. **Stage 1 — discover the STORY** (`news-search`): what happened, when, who, where, and is it
+   worth 60 seconds.
+2. **Stage 2 — find the VISUALS** (`video-search`, `image-search`, the free media catalog): the
+   official footage, the press conference, the eyewitness clip, the generic B-roll that fills gaps.
+
+Searching for news and footage **at the same time** produces poor results for both. Keep the stages apart.
+
+> ## ⚠️ Read this first
+>
+> **A publicly reachable video is not a licensed video.** A TikTok, a YouTube upload, or a news
+> clip appearing in a search result grants you nothing. TikTok supports **embedding** an original
+> post with attribution; downloading and republishing it needs permission or a defensible
+> copyright exception. Treat every link as unlicensed until you check.
+>
+> For commercial client work, in order of preference: **public domain → CC0 → CC BY (with credit)
+> → stock with an explicit commercial licence → written permission from the creator.** Everything
+> else is a lead, not an asset.
+>
+> **This is a GENERAL METHOD, not a fixed script.** The queries below are formulas. Re-derive the
+> exact strings for your topic, region, and moment.
+
+---
+
+## The three calls
+
+All three are synchronous, **paid plans only**, and cost a flat **$0.0003 per call** whatever the
+result count — so ask for a **wide page once** rather than paging twice.
+
+```bash
+vidfarm news-search "AI video startup funding" --fresh w --limit 25
+vidfarm video-search "warehouse robot demonstration footage" --limit 40
+vidfarm image-search "Manila flooding press conference" --limit 40
+```
+
+REST twins: `GET /api/v1/primitives/news-search`, `/video-search`, `/image-search`
+(`?q=…&max_results=…&region=…&timelimit=…`). POST with a JSON body works identically.
+
+---
+
+## Stage 1 — discover the story
+
+### Query formula
+
+```
+[subject] + [event/action] + [location] + [freshness clue]
+```
+
+Examples:
+
+```
+AI startup funding announced today
+factory opening Philippines
+viral product launch this week
+robot delivery testing Toronto
+TikTok creator economy latest
+```
+
+Narrow the time window with `--fresh d|w|m|y` (`timelimit`) rather than words like "today" —
+the freshness parameter is reliable, the word is not.
+
+### Operators that work
+
+```
+"exact phrase"
+site:domain.com
+-keyword
+-site:domain.com
+OR
+```
+
+```
+"AI video editing" startup
+"virtual assistants" Philippines -jobs
+TikTok creator fund OR monetization
+OpenAI video announcement -Reddit
+```
+
+### Trusted sources
+
+```
+site:reuters.com artificial intelligence video
+site:apnews.com Philippines technology
+site:techcrunch.com creator economy
+site:theverge.com TikTok editing
+site:newsroom.tiktok.com creators
+```
+
+### Primary sources (where the real footage usually is)
+
+```
+site:youtube.com official [event]
+site:newsroom.company.com [event]
+site:gov.ph [event]
+site:*.gov press conference [topic]
+```
+
+### What to collect per story
+
+- Headline
+- Publication time **and** event date (they differ, and the event date is the one that matters)
+- People / company / location
+- Primary source URL
+- **Two independent** reporting sources
+
+---
+
+## Stage 2 — find the visuals
+
+### Stories that come with video
+
+Add media words to the story, not to the topic:
+
+```
+[story] video
+[story] footage
+[story] caught on camera
+[story] press conference
+[story] demonstration
+[story] eyewitness video
+[story] livestream
+[story] official footage
+```
+
+```
+Manila flooding eyewitness video
+warehouse robot demonstration footage
+new smartphone launch press conference
+```
+
+**Local TV stations often carry better raw visuals than large written publications:**
+
+```
+[location] [event] local news video
+[location] [event] TV footage
+site:youtube.com [location] [event] news
+```
+
+### The four searches to run for every story
+
+```
+"[company or event]" official video
+"[person]" press conference footage
+"[location]" raw footage
+site:youtube.com "[exact event]"
+site:tiktok.com "[exact event]"
+site:pexels.com/videos [generic visual]
+```
+
+### TikTok
+
+Regular Google search beats Google News for TikTok:
+
+```
+site:tiktok.com "exact event"
+site:tiktok.com/@*/video/ "company name"
+site:tiktok.com "Manila flooding" today
+site:tiktok.com "new robot" demonstration
+```
+
+Use a TikTok as: a **lead** pointing at an event · a **social reaction** shown through the official
+embed · a **style reference** for the edit · **footage only** with the creator's permission.
+
+Google does not index every TikTok. When you want breadth, TikTok's own in-app search is better;
+Google wins when you want one exact phrase or creator.
+
+### Filling the gaps
+
+Whatever the story does not supply, take from licensed stock — the free catalog first:
+
+```bash
+vidfarm media search "city traffic night" --type video     # Pixabay / Openverse, $0, licence attached
+vidfarm public-raws --category b-roll                       # the platform's own free shelf
+```
+
+---
+
+## The pipeline
+
+```
+news-search  →  video-search / image-search  →  licensed stock fills the gaps
+  (story)          (original visuals)              (generic shots)
+```
+
+Then the normal Vidfarm path: `vidfarm raws scan <url>` to mine clips out of a source you are
+entitled to use, `vidfarm download-video <url>` to collect one file, fork a template, edit, render.
+
+---
+
+## Copy-paste agent prompt
+
+```
+Search Google News for stories about [TOPIC] published within the last [TIME WINDOW].
+
+Prioritize:
+1. Stories with strong visual potential
+2. Events with official videos, demonstrations, press conferences or eyewitness footage
+3. Primary sources and at least two independent reports
+4. Stories that can be explained accurately in under 60 seconds
+
+For each story, return:
+- Headline and event date
+- One-sentence summary
+- Why it is visually interesting
+- Primary source
+- Two corroborating sources
+- Five suggested B-roll searches
+- Official footage links, if available
+- Relevant YouTube and TikTok search queries
+- Footage licensing or permission status
+
+Do not treat a publicly accessible video as licensed for reuse.
+```
+
+---
+
+## Accuracy rules for the video itself
+
+Timely content is the fastest way to be publicly wrong. Three rules:
+
+1. **Two independent sources or it does not go on screen.** One outlet reporting a claim is a
+   claim, not a fact.
+2. **Date the event on screen** when the story is developing. "As of [date]" costs one line and
+   protects the video when the story moves.
+3. **Attribute footage in-frame** when the licence asks for it, and never imply an organisation or
+   person endorses you because their clip appears.
+
+The hook/loop/payoff standard still applies — a news peg is a **reason to watch now**, not a
+substitute for a hook.