eleventy-plugin-podcaster 2.0.0-alpha.2 → 2.0.0-alpha.4

package/.env

@@ -0,0 +1,5 @@
+BUCKET_SECRET_KEY="GiBsTUbU7VwKdSZGZKvYb47i87lWjkUXRH1Edd+I2KQ"
+BUCKET_ACCESS_KEY="DO801WKX4LJP3Y9DWE2H"
+BUCKET_ENDPOINT="https://sfo3.digitaloceanspaces.com"
+BUCKET_REGION="us-east-1"
+BUCKET_NAME="startlingbarbarabain"

package/eleventy.config.js

@@ -1,17 +1,27 @@
+import path from 'node:path'
+
 import podcastFeed from './src/podcastFeed.js'
 import podcastData from './src/podcastData.js'
 import episodeData from './src/episodeData.js'
-import calculateFilenameSizeAndDuration from './src/calculateFilenameSizeAndDuration.js'
+import calculateEpisodeSizeAndDuration from './src/calculateEpisodeSizeAndDuration.js'
+import calculateEpisodeFilename from './src/calculateEpisodeFilename.js'
+
 import readableFilters from './src/readableFilters.js'
 import excerpts from './src/excerpts.js'
 import drafts from './src/drafts.js'
 import pageTitle from './src/pageTitle.js'
 
 export default function (eleventyConfig, options = {}) {
+  const episodePostsDirectory = options.episodePostsDirectory ?? 'episode-posts'
+  options.episodePostsDirectory = path.join(eleventyConfig.directories.input, episodePostsDirectory)
+  const episodeFilesDirectory = options.episodeFilesDirectory ?? 'episode-files'
+  options.episodeFilesDirectory = path.join(eleventyConfig.directories.input, episodeFilesDirectory)
+
   eleventyConfig.addPlugin(podcastFeed, options)
   eleventyConfig.addPlugin(podcastData, options)
   eleventyConfig.addPlugin(episodeData, options)
-  eleventyConfig.addPlugin(calculateFilenameSizeAndDuration, options)
+  eleventyConfig.addPlugin(calculateEpisodeSizeAndDuration, options)
+  eleventyConfig.addPlugin(calculateEpisodeFilename, options)
 
   // Filters
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eleventy-plugin-podcaster",
-  "version": "2.0.0-alpha.2",
+  "version": "2.0.0-alpha.4",
   "description": "An Eleventy plugin that allows you to create a podcast and its accompanying website",
   "main": "eleventy.config.js",
   "exports": {
@@ -30,8 +30,8 @@
   "dependencies": {
     "@11ty/eleventy": "^3.0.0",
     "@11ty/eleventy-plugin-rss": "^2.0.1",
+    "@aws-sdk/client-s3": "^3.862.0",
     "@tsmx/human-readable": "^2.0.3",
-    "chalk": "^5.3.0",
     "dom-serializer": "^2.0.0",
     "htmlparser2": "^9.1.0",
     "luxon": "^3.4.4",
@@ -40,7 +40,9 @@
   },
   "devDependencies": {
     "ava": "^6.1.3",
+    "dotenv": "^17.2.1",
     "fast-xml-parser": "^4.4.0",
+    "mock-aws-s3-v3": "^6.0.5",
     "neostandard": "^0.11.4"
   }
 }

package/src/calculateEpisodeFilename.js

@@ -0,0 +1,41 @@
+import isEpisodePost from './isEpisodePost.js'
+
+function findMatchingFilename (episodeData, thisEpisode) {
+  const filenameSeasonAndEpisodePattern =
+    /^.*?\b[sS](?<seasonNumber>\d+)\s*[eE](?<episodeNumber>\d+)\b.*\.(mp3|m4a)$/
+  const filenameEpisodePattern = /^.*?\b(?<episodeNumber>\d+)\b.*\.(mp3|m4a)$/
+  const { seasonNumber, episodeNumber } = thisEpisode
+
+  for (const file of Object.keys(episodeData)) {
+    if (seasonNumber && episodeNumber) {
+      const seasonAndEpisodeMatch = file.match(filenameSeasonAndEpisodePattern)
+      if (seasonAndEpisodeMatch) {
+        const matchedSeasonNumber = parseInt(seasonAndEpisodeMatch.groups.seasonNumber)
+        const matchedEpisodeNumber = parseInt(seasonAndEpisodeMatch.groups.episodeNumber)
+        if (matchedSeasonNumber === seasonNumber &&
+            matchedEpisodeNumber === episodeNumber) {
+          return file
+        }
+      }
+    } else if (episodeNumber) {
+      const episodeMatch = file.match(filenameEpisodePattern)
+      if (episodeMatch) {
+        const matchedEpisodeNumber = parseInt(episodeMatch.groups.episodeNumber)
+        if (matchedEpisodeNumber === episodeNumber) {
+          return file
+        }
+      }
+    }
+  }
+}
+
+export default function (eleventyConfig, options) {
+  eleventyConfig.addGlobalData('eleventyComputed.episode.filename', () => {
+    return data => {
+      if (data.episode.filename) return data.episode.filename
+      if (!isEpisodePost(data, options)) return
+
+      return findMatchingFilename(data.episodeData, data.episode)
+    }
+  })
+}

package/src/calculateEpisodeSizeAndDuration.js

@@ -0,0 +1,207 @@
+import { Duration, DateTime } from 'luxon'
+import path from 'node:path'
+import { existsSync } from 'node:fs'
+import { readdir, stat, writeFile } from 'node:fs/promises'
+import { Writable } from 'node:stream'
+import { S3Client, ListObjectsCommand, GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3'
+import { parseFile as parseFileMetadata, parseBuffer as parseBufferMetadata } from 'music-metadata'
+import hr from '@tsmx/human-readable'
+import isEpisodePost from './isEpisodePost.js'
+
+const isAudioFile = episodeFilename => episodeFilename.endsWith('.mp3') ||
+                    episodeFilename.endsWith('.m4a')
+
+const convertSecondsToReadableDuration = seconds =>
+  Duration.fromMillis(seconds * 1000)
+    .shiftTo('days', 'hours', 'minutes', 'seconds')
+    .toHuman()
+
+const convertReadableDurationToSeconds = duration => {
+  const durationPattern = /^(?:(?<hours>\d+):)?(?<minutes>\d{1,2}):(?<seconds>\d{2}(?:\.\d+)?)$/
+
+  let match
+  if (duration?.match) {
+    match = duration.match(durationPattern)
+  }
+
+  if (match) {
+    const hours = isNaN(parseInt(match.groups.hours))
+      ? 0
+      : parseInt(match.groups.hours)
+    const minutes = parseInt(match.groups.minutes)
+    const seconds = parseFloat(match.groups.seconds)
+    return hours * 3600 + minutes * 60 + seconds
+  }
+}
+
+async function readEpisodeDataLocally (episodeFilesDirectory) {
+  const episodes = await readdir(episodeFilesDirectory)
+  const episodeData = {}
+  for (const episode of episodes) {
+    if (!isAudioFile(episode)) continue
+
+    const episodePath = path.join(episodeFilesDirectory, episode)
+    const episodeSize = (await stat(episodePath)).size
+    const episodeMetadata = await parseFileMetadata(episodePath, { duration: true })
+    const episodeDuration = episodeMetadata.format.duration
+    episodeData[episode] = {
+      size: episodeSize,
+      duration: episodeDuration
+    }
+  }
+  return episodeData
+}
+
+function calculatePodcastData (episodeData) {
+  const episodeDataValues = Object.values(episodeData)
+  const numberOfEpisodes = episodeDataValues.length
+  const totalSize = episodeDataValues.map(x => x.size).reduce((x, y) => x + y)
+  const totalDuration = episodeDataValues.map(x => x.duration).reduce((x, y) => x + y)
+  return { numberOfEpisodes, totalSize, totalDuration }
+}
+
+function reportPodcastData (podcastData) {
+  const { numberOfEpisodes, totalSize, totalDuration } = podcastData
+  console.log(`\u001b[33m${numberOfEpisodes} episodes; ${hr.fromBytes(totalSize)}; ${convertSecondsToReadableDuration(totalDuration)}.\u001b[0m`)
+}
+
+async function writePodcastDataLocally (episodeData, podcastData, directories) {
+  const dataDir = path.join(process.cwd(), directories.data)
+  await writeFile(path.join(dataDir, 'episodeData.json'), JSON.stringify(episodeData, null, 2))
+  await writeFile(path.join(dataDir, 'podcastData.json'), JSON.stringify(podcastData, null, 2))
+}
+
+function getS3Client (options) {
+  if (options.s3ClientObject) return options.s3ClientObject
+
+  if (options.s3Client) {
+    return new S3Client({
+      forcePathStyle: true,
+      endpoint: options.s3Client.endpoint,
+      region: options.s3Client.region,
+      credentials: {
+        accessKeyId: options.s3Client.accessKey,
+        secretAccessKey: options.s3Client.secretKey
+      }
+    })
+  }
+}
+
+async function getObjectFromS3Bucket (s3Client, s3Bucket, key) {
+  const getObjectResponse = await s3Client.send(new GetObjectCommand({ Bucket: s3Bucket, Key: key }))
+
+  const chunks = []
+  if (typeof getObjectResponse.Body.pipe === 'function') {
+    // this is to cope with the behaviour of the mock, which doesn't return an iterator full of chunks
+    const writable = new Writable({
+      write (chunk, encoding, callback) {
+        chunks.push(chunk)
+        callback()
+      },
+      final (callback) {
+        callback()
+      }
+    })
+    getObjectResponse.Body.pipe(writable)
+    await new Promise((resolve, reject) => {
+      writable.on('finish', resolve)
+      writable.on('error', reject)
+    })
+  } else {
+    for await (const chunk of getObjectResponse.Body) {
+      chunks.push(chunk)
+    }
+  }
+  const buffer = Buffer.concat(chunks)
+  return { buffer, lastModified: getObjectResponse.LastModified }
+}
+
+async function getStoredEpisodeDataFromS3Bucket (s3Client, s3BucketName) {
+  try {
+    const { buffer, lastModified } = await getObjectFromS3Bucket(s3Client, s3BucketName, 'episodeData.json')
+    return { episodeData: JSON.parse(buffer.toString()), lastModified }
+  } catch (err) {
+    return { episodeData: {}, lastModified: null }
+  }
+}
+
+async function updateEpisodeDataFromS3Bucket (s3Client, s3Bucket) {
+  const storedEpisodeData = await getStoredEpisodeDataFromS3Bucket(s3Client, s3Bucket)
+  const storedEpisodeDataLastModifiedDate = (storedEpisodeData.lastModified)
+    ? DateTime.fromISO(storedEpisodeData.lastModified)
+    : null
+  const list = await s3Client.send(new ListObjectsCommand({ Bucket: s3Bucket }))
+  const result = { ...storedEpisodeData.episodeData }
+  for (const item of list.Contents ?? []) {
+    if (!isAudioFile(item.Key)) continue
+
+    const { Key: filename, Size: size, LastModified: lastModified } = item
+
+    if (!(filename in result) ||
+        !('size' in result[filename]) ||
+        !('duration' in result[filename]) ||
+        !storedEpisodeDataLastModifiedDate ||
+        storedEpisodeDataLastModifiedDate > DateTime.fromISO(lastModified)) {
+      const { buffer } = await getObjectFromS3Bucket(s3Client, s3Bucket, filename)
+      const metadata = await parseBufferMetadata(buffer, null, { duration: true })
+      const duration = metadata.format.duration
+      result[filename] = { size, duration }
+    }
+  }
+  return result
+}
+
+async function writeEpisodeDataToS3Bucket (s3Client, s3Bucket, episodeData) {
+  await s3Client.send(new PutObjectCommand({
+    Bucket: s3Bucket,
+    Key: 'episodeData.json',
+    Body: JSON.stringify(episodeData, null, 2),
+    ContentType: 'application/json'
+  }))
+}
+
+export default function (eleventyConfig, options = {}) {
+  let firstRun = true
+  eleventyConfig.on('eleventy.before', async ({ directories }) => {
+    if (!firstRun || process.env.SKIP_EPISODE_CALCULATIONS === 'true') return
+    firstRun = false
+
+    const episodeFilesDirectory = options.episodeFilesDirectory
+    let episodeData
+    if (existsSync(episodeFilesDirectory)) {
+      episodeData = await readEpisodeDataLocally(episodeFilesDirectory)
+    } else if (options.s3ClientObject || options.s3Client) {
+      const s3Client = getS3Client(options)
+      const s3Bucket = options.s3Client.bucket
+      episodeData = await updateEpisodeDataFromS3Bucket(s3Client, s3Bucket)
+      await writeEpisodeDataToS3Bucket(s3Client, s3Bucket, episodeData)
+    } else {
+      return
+    }
+    const podcastData = calculatePodcastData(episodeData)
+    await writePodcastDataLocally(episodeData, podcastData, directories)
+    if (!eleventyConfig.quietMode) reportPodcastData(podcastData)
+  })
+
+  eleventyConfig.addGlobalData('eleventyComputed.episode.size', () => {
+    return data => {
+      if (data.episode.size) return data.episode.size
+      if (isEpisodePost(data, options) && data.episodeData) {
+        return data.episodeData[data.episode.filename]?.size
+      }
+    }
+  })
+
+  eleventyConfig.addGlobalData('eleventyComputed.episode.duration', () => {
+    return data => {
+      if (data.episode.duration) {
+        const convertedReadableDuration = convertReadableDurationToSeconds(data.episode.duration)
+        return convertedReadableDuration ?? data.episode.duration
+      }
+
+      if (isEpisodePost(data, options) && data.episodeData) {
+        return data.episodeData[data.episode.filename]?.duration
+      }
+    }
+  })
+}

package/src/drafts.js

@@ -13,9 +13,9 @@ export default (eleventyConfig, options = {}) => {
     }
     if (!hasLoggedAboutDrafts) {
       if (shouldIncludeDrafts) {
-        console.log('Including drafts.')
+        if (!eleventyConfig.quietMode) console.log('Including drafts.')
       } else {
-        console.log('Excluding drafts.')
+        if (!eleventyConfig.quietMode) console.log('Excluding drafts.')
       }
       hasLoggedAboutDrafts = true
     }

package/src/episodeData.js

@@ -1,3 +1,5 @@
+import isEpisodePost from './isEpisodePost.js'
+
 export default function (eleventyConfig, options = {}) {
   const postFilenameSeasonAndEpisodePattern =
     /^[sS](?<seasonNumber>\d+)[eE](?<episodeNumber>\d+)/i
@@ -7,7 +9,7 @@ export default function (eleventyConfig, options = {}) {
     return data => {
       if (data.episode?.seasonNumber) return data.episode.seasonNumber
 
-      if (!data.page.inputPath.includes('/episodePosts/')) return
+      if (!isEpisodePost(data, options)) return
 
       const seasonAndEpisodeMatch = data.page.fileSlug.match(postFilenameSeasonAndEpisodePattern)
       if (seasonAndEpisodeMatch) {
@@ -20,7 +22,7 @@ export default function (eleventyConfig, options = {}) {
     return data => {
       if (data.episode?.episodeNumber) return data.episode.episodeNumber
 
-      if (!data.page.inputPath.includes('/episodePosts/')) return
+      if (!isEpisodePost(data, options)) return
 
       const seasonAndEpisodeMatch = data.page.fileSlug.match(postFilenameSeasonAndEpisodePattern)
       if (seasonAndEpisodeMatch) {
@@ -49,7 +51,7 @@ export default function (eleventyConfig, options = {}) {
 
   eleventyConfig.addGlobalData('eleventyComputed.episode.url', () => {
     return data => {
-      if (!data.page.inputPath.includes('/episodePosts/')) return
+      if (!isEpisodePost(data, options)) return
 
       const episodeUrlBase = data.podcast.episodeUrlBase
       const filename = data.episode.filename

package/src/excerpts.js

@@ -1,11 +1,12 @@
 import * as htmlparser2 from 'htmlparser2'
 import render from 'dom-serializer'
 import markdownIt from 'markdown-it'
+import isEpisodePost from './isEpisodePost.js'
 
 export default function (eleventyConfig, options = {}) {
   eleventyConfig.addGlobalData('eleventyComputed.excerpt', () => {
     return (data) => {
-      if (!data.page.inputPath.includes('/episodePosts/')) return
+      if (!isEpisodePost(data, options)) return
 
       const md = markdownIt({
         html: true,

package/src/isEpisodePost.js

@@ -0,0 +1,10 @@
+import path from 'node:path'
+
+export default function isEpisodePost (data, options) {
+  if (data.page?.inputPath) {
+    const importPath = path.normalize(data.page.inputPath)
+    return importPath.startsWith(options.episodePostsDirectory)
+  } else {
+    return false
+  }
+}

package/src/podcastFeed.js

@@ -1,6 +1,7 @@
 import rssPlugin from '@11ty/eleventy-plugin-rss'
 import { readFileSync } from 'node:fs'
 import path from 'node:path'
+import isEpisodePost from './isEpisodePost.js'
 
 export default function (eleventyConfig, options = {}) {
   if (!('addTemplate' in eleventyConfig)) {
@@ -12,7 +13,7 @@ export default function (eleventyConfig, options = {}) {
   eleventyConfig.addTemplate('feed.njk', readFileSync(podcastFeedPath), {
     eleventyExcludeFromCollections: true,
     eleventyImport: {
-      collections: ['podcastEpisode']
+      collections: ['episodePost']
     }
   })
 
@@ -22,7 +23,7 @@ export default function (eleventyConfig, options = {}) {
     }
   })
 
-  eleventyConfig.addCollection('podcastEpisode', (collectionApi) => {
-    return collectionApi.getFilteredByGlob('**/episodePosts/*')
+  eleventyConfig.addCollection('episodePost', (collectionApi) => {
+    return collectionApi.getAll().filter(item => isEpisodePost(item.data, options))
   })
 }

package/src/podcastFeed.njk

@@ -17,7 +17,7 @@ eleventyAllowMissingExtension: true
     <description>{{ podcast.description }}</description>
     <language>{{ podcast.language }}</language>
     <copyright>{{ podcast.copyrightNotice }}</copyright>
-    <pubDate>{{ collections.podcastEpisode | getNewestCollectionItemDate | dateToRfc3339 }}</pubDate>
+    <pubDate>{{ collections.episodePost | getNewestCollectionItemDate | dateToRfc3339 }}</pubDate>
     <lastBuildDate>{{ podcast.feedLastBuildDate }}</lastBuildDate>
     <itunes:image href="{{ podcast.imagePath | htmlBaseUrl(siteUrl) }}"></itunes:image>
     {%- if podcast.subcategory %}
@@ -47,7 +47,7 @@ eleventyAllowMissingExtension: true
     </itunes:owner>
     {%- endif %}
 
-    {% for post in collections.podcastEpisode | reverse %}
+    {% for post in collections.episodePost | reverse %}
     <item>
       <title>{{ post.data.episode.title or post.data.title }}</title>
       {% if post.data.episode.itunesTitle %}

package/src/calculateFilenameSizeAndDuration.js

@@ -1,106 +0,0 @@
-import { Duration } from 'luxon'
-import path from 'node:path'
-import { existsSync } from 'node:fs'
-import { readdir, stat, writeFile } from 'node:fs/promises'
-import { parseFile } from 'music-metadata'
-import hr from '@tsmx/human-readable'
-import chalk from 'chalk'
-
-const convertSecondsToReadableDuration = seconds =>
-  Duration.fromMillis(seconds * 1000)
-    .shiftTo('days', 'hours', 'minutes', 'seconds')
-    .toHuman()
-
-export default function (eleventyConfig) {
-  let firstRun = true
-  eleventyConfig.on('eleventy.before', async ({ directories }) => {
-    // don't keep recalculating episode data in serve mode
-    if (!firstRun || process.env.SKIP_EPISODE_CALCULATIONS === 'true') return
-    firstRun = false
-    const episodesDir = path.join(directories.input, 'episodeFiles')
-    if (!existsSync(episodesDir)) return
-
-    const episodes = await readdir(episodesDir)
-    const episodeData = {}
-    let numberOfEpisodes = 0
-    let totalSize = 0
-    let totalDuration = 0
-
-    for (const episode of episodes) {
-      if (!episode.endsWith('.mp3')) continue
-
-      numberOfEpisodes++
-      const episodePath = path.join(episodesDir, episode)
-      const episodeSize = (await stat(episodePath)).size
-      totalSize += episodeSize
-      const episodeMetadata = await parseFile(episodePath, { duration: true })
-      const episodeDuration = episodeMetadata.format.duration
-      totalDuration += episodeDuration
-      episodeData[episode] = {
-        size: episodeSize,
-        duration: Math.round(episodeDuration * 1000) / 1000
-      }
-      totalDuration = Math.round(totalDuration * 1000) / 1000
-    }
-    const podcastData = { numberOfEpisodes, totalSize, totalDuration }
-
-    const dataDir = path.join(process.cwd(), directories.data)
-    await writeFile(path.join(dataDir, 'episodeData.json'), JSON.stringify(episodeData, null, 2))
-    await writeFile(path.join(dataDir, 'podcastData.json'), JSON.stringify(podcastData, null, 2))
-
-    console.log(chalk.yellow(`${numberOfEpisodes} episodes; ${hr.fromBytes(totalSize)}; ${convertSecondsToReadableDuration(totalDuration)}.`))
-  })
-
-  const filenameSeasonAndEpisodePattern =
-    /^.*?\b[sS](?<seasonNumber>\d+)\s*[eE](?<episodeNumber>\d+)\b.*\.mp3$/
-  const filenameEpisodePattern = /^.*?\b(?<episodeNumber>\d+)\b.*\.mp3$/
-
-  eleventyConfig.addGlobalData('eleventyComputed.episode.filename', () => {
-    return data => {
-      if (data.episode.filename) return data.episode.filename
-
-      if (!data.page.inputPath.includes('/episodePosts/')) return
-
-      for (const file of Object.keys(data.episodeData)) {
-        if (data.episode.seasonNumber && data.episode.episodeNumber) {
-          const seasonAndEpisodeMatch = file.match(filenameSeasonAndEpisodePattern)
-          if (seasonAndEpisodeMatch) {
-            const matchedSeasonNumber = parseInt(seasonAndEpisodeMatch.groups.seasonNumber)
-            const matchedEpisodeNumber = parseInt(seasonAndEpisodeMatch.groups.episodeNumber)
-            if (matchedSeasonNumber === data.episode.seasonNumber &&
-                matchedEpisodeNumber === data.episode.episodeNumber) {
-              return file
-            }
-          }
-        } else if (data.episode.episodeNumber) {
-          const episodeMatch = file.match(filenameEpisodePattern)
-          if (episodeMatch) {
-            const matchedEpisodeNumber = parseInt(episodeMatch.groups.episodeNumber)
-            if (matchedEpisodeNumber === data.episode.episodeNumber) {
-              return file
-            }
-          }
-        }
-      }
-    }
-  })
-
-  eleventyConfig.addGlobalData('eleventyComputed.episode.size', () => {
-    return data => {
-      if (data.episode.size) return data.episode.size
-      if (data.page.inputPath.includes('/episodePosts/') && data.episodeData) {
-        return data.episodeData[data.episode.filename]?.size
-      }
-    }
-  })
-
-  eleventyConfig.addGlobalData('eleventyComputed.episode.duration', () => {
-    return data => {
-      if (data.episode.duration) return data.episode.duration
-
-      if (data.page.inputPath.includes('/episodePosts/') && data.episodeData) {
-        return data.episodeData[data.episode.filename]?.duration
-      }
-    }
-  })
-}