markdown-magic 3.0.7 → 3.0.9

package/lib/transforms/{code.js → code/index.js}

@@ -1,9 +1,13 @@
 const fs = require('fs')
 const path = require('path')
-const remoteRequest = require('../utils/remoteRequest')
-const { isLocalPath } = require('../utils/fs')
-const { deepLog } = require('../utils/logs')
-const { getLineCount, getTextBetweenLines } = require('../utils/text')
+const remoteRequest = require('../../utils/remoteRequest')
+const { isLocalPath } = require('../../utils/fs')
+const { deepLog } = require('../../utils/logs')
+const { getLineCount, getTextBetweenLines } = require('../../utils/text')
+const { resolveGithubContents, isGithubLink } = require('./resolve-github-file')
+
+const GITHUB_LINK = /https:\/\/github\.com\/([^/\s]*)\/([^/\s]*)\/blob\//
+const GIST_LINK = /https:\/\/gist\.github\.com\/([^/\s]*)\/([^/\s]*)(\/)?/
 
 /**
  * Options for specifying source code to include in documentation.
@@ -26,19 +30,26 @@ const { getLineCount, getTextBetweenLines } = require('../utils/text')
    ```
  */
 
+
 // TODO code sections 
 // https://github.com/linear/linear/blob/94af540244864fbe466fb933256278e04e87513e/docs/transforms/code-section.js
 // https://github.com/linear/linear/blob/bc39d23af232f9fdbe7df458b0aaa9554ca83c57/packages/sdk/src/_tests/readme.test.ts#L133-L140
 // usage https://github.com/linear/linear/blame/93981d3a3db571e2f8efdce9f5271ea678941c43/packages/sdk/README.md#L1
 
-module.exports = function CODE(api) {
+module.exports = async function CODE(api) {
   const { content, srcPath } = api
   /** @type {CodeTransformOptions} */
   const options = api.options || {}
   // console.log('CODE API', api)
   // process.exit(1)
-  // console.log('options', options)
-  const { src, lines, id } = options
+  const { 
+    id,
+    lines,
+    isPrivate,
+    accessToken
+  } = options
+
+  let src = options.src
   const originalContent = content
   let code
   let syntax = options.syntax
@@ -61,20 +72,43 @@ module.exports = function CODE(api) {
       syntax = path.extname(codeFilePath).replace(/^./, '')
     }
   } else {
-    // do remote request
-    // console.log(src)
-    const remoteContent = remoteRequest(src)
+    /* Automatically get raw code files from github */
+    // Convert https://github.com/DavidWells/markdown-magic/blob/master/package.json
+    // to https://raw.githubusercontent.com/DavidWells/markdown-magic/master/package.json
+
+    if (src.match(GITHUB_LINK)) {
+      src = src.replace(GITHUB_LINK, 'https://raw.githubusercontent.com/$1/$2/')
+    }
+    /* Automatically get raw code files from gist... needs api call.... */
+    // https://gist.github.com/DavidWells/7d2e0e1bc78f4ac59a123ddf8b74932d
+    // https://gist.githubusercontent.com/DavidWells/7d2e0e1bc78f4ac59a123ddf8b74932d/raw/0808a83de7f07c931fb81ed691c1d6bbafad29d1/aligning-images.md
+
+    let remoteContent
+
+    if (isGithubLink(src)) {
+      remoteContent = await resolveGithubContents({
+        repoFilePath: src,
+        accessToken,
+        // debug: true
+      })
+    }
+
+    // Try initial remote request if public url
+    if (!remoteContent) {
+      remoteContent = remoteRequest(src)
+    }
+  
     if (!remoteContent) {
-      console.log(`WARNING: ${src} URL NOT FOUND  or internet connection is off`)
+      console.log(`WARNING: ${src} URL NOT FOUND or internet connection is off or no access to remove URL`)
       return originalContent
     }
     code = remoteContent
-    syntax = path.extname(src).replace(/^./, '')
+    syntax = (path.extname(src).replace(/^./, '') || '').split('#')[0]
   }
 
   /* handle option `lines` */
   if (options.lines) {
-    const lineCount = getLineCount(code)
+    // const lineCount = getLineCount(code)
     // console.log('lineCount', lineCount)
     // console.log('src', src)
     // console.log('lines', lines)

package/lib/transforms/code/resolve-github-file.js

@@ -0,0 +1,362 @@
+const https = require('https')
+const { exec } = require('child_process')
+const { getTextBetweenLines } = require('../../utils/text')
+
+const VALID_SLUG_REGEX = /^[A-Z-a-z0-9_-]*$/
+const VALID_FILE_REGEX = /^[^;]*$/
+const GITHUB_LINK_REGEX = /^(?:https:\/\/)?github\.com\/([^/\s]*)\/([^/\s]*)\/blob\/([^/\s]*)\/([^\s]*)/
+const GITHUB_RAW_LINK_REGEX = /^(?:https:\/\/)?raw\.githubusercontent\.com\/([^/\s]*)\/([^/\s]*)\/([^/\s]*)\/([^\s]*)/
+
+function isGithubLink(str = '') {
+  return isGithubRepoLink(str) || isGithubRawLink(str)
+}
+
+function isGithubRepoLink(str = '') {
+  return GITHUB_LINK_REGEX.test(str)
+}
+
+function isGithubRawLink(str = '') {
+  return GITHUB_RAW_LINK_REGEX.test(str)
+}
+
+function convertLinkToRaw(link) {
+  if (!isGithubRepoLink(link)) return link
+  return link.replace(GITHUB_LINK_REGEX, 'https://raw.githubusercontent.com/$1/$2/$3/$4')
+}
+
+function resolveGithubDetails(repoFilePath) {
+  let parts
+  if (isGithubRepoLink(repoFilePath)) {
+    parts = repoFilePath.match(GITHUB_LINK_REGEX)
+  }
+  if (isGithubRawLink(repoFilePath)) {
+    parts = repoFilePath.match(GITHUB_RAW_LINK_REGEX)
+  }
+  if (!parts) {
+    return
+  }
+  const [ _match, repoOwner, repoName, branchOrRef, filePath ] = parts
+  const [ filePathStart, hash ] = filePath.split('#')
+  const result = {
+    repoOwner,
+    repoName,
+    filePath: filePathStart,
+  }
+  if (isGitHash(branchOrRef)) {
+    result.ref = branchOrRef
+  } else {
+    result.branch = branchOrRef
+  }
+  if (hash) {
+    const range = parseLineRange(`#${hash}`)
+    if (range) {
+      result.range = range
+    }
+  }
+  return result
+}
+
+/**
+ * Resolves the contents of a file from a GitHub repository.
+ *
+ * @param {Object}  options - The options for resolving the GitHub contents.
+ * @param {string}  options.repoFilePath - The file path in the GitHub repository.
+ * @param {string}  [options.accessToken] - The access token for authenticating with GitHub (optional).
+ * @param {boolean} [options.debug = false] - Whether to enable debug logging (optional).
+ * @returns {Promise<string>} - A promise that resolves to the contents of the file.
+ * @throws {Error} - If the GitHub link is invalid or if the file fetch fails.
+ */
+async function resolveGithubContents({
+  repoFilePath,
+  accessToken,
+  debug = false
+}) {
+  const token = resolveAccessToken(accessToken)
+  const logger = (debug) ? console.log : () => {}
+  const githubDetails = resolveGithubDetails(repoFilePath)
+  if (!githubDetails) {
+    throw new Error(`Invalid github link. "${repoFilePath}" is not a valid github link`)
+  }
+
+  logger(`Github File Details "${repoFilePath}"`, githubDetails)
+
+  const payload = {
+    ...githubDetails,
+    accessToken: token
+  }
+
+  let errs = []
+
+  /* Try raw request first */
+  try {
+    const fileContent = await getGitHubFileContentsRaw(payload)
+    logger(`✅  GitHub file resolved via raw GET`)
+    return returnCode(fileContent, githubDetails.range)
+  } catch (err) {
+    logger('❌  Unable to resolve GitHub raw content')
+    errs.push(err)
+  }
+
+  /* Then try Github CLI or GitHub API */
+  const githubFetcher = (!token) ? getGitHubFileContentsCli : getGitHubFileContentsApi
+  try {
+    const fileContent = await githubFetcher(payload)
+    logger(`✅  GitHub file resolved via ${githubFetcher.name}`)
+    return returnCode(fileContent, githubDetails.range)
+  } catch (err) {
+    logger(`❌ Unable to resolve GitHub file via ${githubFetcher.name}`)
+    errs.push(err)
+  }
+
+  /* Then try API */
+  try {
+    const fileContent = await getGitHubFileContentsApi(payload)
+    logger(`✅  GitHub file resolved via ${getGitHubFileContentsApi.name}`)
+    return returnCode(fileContent, githubDetails.range)
+  } catch (err) {
+    logger(`❌ Unable to resolve GitHub file via ${githubFetcher.name}`)
+    errs.push(err)
+  }
+
+  throw new Error(`Failed to fetch GitHub file "${repoFilePath}". \n${errs.forEach(err => err.message)}`)
+}
+
+function returnCode(fileContent, lines) {
+  if (!lines) return fileContent
+  const [startLine, endLine] =lines
+  return getTextBetweenLines(fileContent, startLine, endLine)
+}
+
+/**
+ * Retrieves the contents of a file from a GitHub repository using the GitHub CLI.
+ *
+ * @param {Object} options - The options for retrieving the file contents.
+ * @param {string} options.repoOwner - The owner of the GitHub repository.
+ * @param {string} options.repoName - The name of the GitHub repository.
+ * @param {string} options.filePath - The path to the file in the repository.
+ * @param {string} [options.branch] - The branch name of the repository.
+ * @param {string} [options.ref] - The ref of the repository.
+ * @returns {Promise<string>} A promise that resolves with the decoded content of the file.
+ * @throws {Error} If there is an error retrieving the file contents.
+ */
+async function getGitHubFileContentsCli(options) {
+  validateInputs(options)
+  const {
+    repoOwner, 
+    repoName,
+    filePath, 
+    branch,
+    ref,
+  } = options
+
+  let flags = ''
+  if (ref) {
+    flags = `?ref=${ref}`
+  }
+  if (branch) {
+    flags = `?ref=${branch}`
+  }
+  const command = `gh api repos/${repoOwner}/${repoName}/contents/${filePath}${flags}`
+  /*
+  console.log('command', command)
+  /** */
+  return new Promise((resolve, reject) => {
+    exec(command, (error, stdout, stderr) => {
+      if (error) {
+        return reject(error)
+      }
+      const fileContent = JSON.parse(stdout).content;
+      const decodedContent = decode(fileContent)
+      return resolve(decodedContent)
+    })
+  })
+}
+
+/**
+ * Retrieves the contents of a file from a GitHub repository via the github API
+ *
+ * @param {Object} options - The options for retrieving the file contents.
+ * @param {string} options.repoOwner - The owner of the GitHub repository.
+ * @param {string} options.repoName - The name of the GitHub repository.
+ * @param {string} options.filePath - The path to the file in the repository.
+ * @param {string} [options.branch] - The branch name to fetch the file from. If not provided, the default branch will be used.
+ * @param {string} [options.ref] - The ref (commit SHA or branch name) to fetch the file from. If provided, it takes precedence over the branch.
+ * @param {string} [options.accessToken] - The access token for authenticating the request (optional).
+ * @returns {Promise<string>} A promise that resolves with the decoded contents of the file.
+ * @throws {Error} If the file retrieval fails or the response status code is not 200.
+ */
+function getGitHubFileContentsApi(options) {
+  validateInputs(options)
+
+  const {
+    repoOwner, 
+    repoName,
+    filePath, 
+    branch,
+    ref,
+    accessToken
+  } = options
+
+  let flags = ''
+  if (ref) {
+    flags = `?ref=${ref}`
+  }
+  if (branch) {
+    flags = `?branch=${branch}`
+  }
+  const apiEndpoint = `/repos/${repoOwner}/${repoName}/contents/${filePath}${flags}`
+  /*
+  // console.log('apiEndpoint', apiEndpoint)
+  /** */
+  return new Promise((resolve, reject) => {
+    const options = {
+      hostname: 'api.github.com',
+      path: apiEndpoint,
+      method: 'GET',
+      headers: {
+        'User-Agent': 'Node.js',
+        ...(accessToken) ? { 'Authorization': `token ${accessToken}` } : {},
+      }
+    }
+    /*
+    console.log('getGitHubFileContentsApi options', options)
+    /** */
+    const req = https.request(options, (res) => {
+      let data = ''
+      res.on('data', (chunk) => {
+        data += chunk
+      })
+      res.on('end', () => {
+        if (res.statusCode === 200) {
+          const fileContent = JSON.parse(data).content
+          const decodedContent = decode(fileContent)
+          resolve(decodedContent)
+        } else {
+          reject(new Error(`Failed to fetch file. Status code: ${res.statusCode}`))
+        }
+      })
+    })
+
+    req.on('error', (error) => {
+      reject(error)
+    })
+
+    req.end()
+  })
+}
+
+function getGitHubFileContentsRaw(options) {
+  validateInputs(options)
+
+  const {
+    repoOwner,
+    repoName,
+    filePath,
+    branch,
+    accessToken
+  } = options
+  
+  const [ _filePath, hash ] = filePath.split('#')
+  return new Promise((resolve, reject) => {
+    const options = {
+      hostname: 'raw.githubusercontent.com',
+      path: `/${repoOwner}/${repoName}/${branch}/${_filePath}`,
+      method: 'GET',
+      headers: {
+        'User-Agent': 'Node.js',
+        ...(accessToken) ? { 'Authorization': `token ${accessToken}` } : {},
+      }
+    }
+    /*
+    console.log('getGitHubFileContentsRaw options', options)
+    /** */
+    const req = https.request(options, (res) => {
+      let data = ''
+      res.on('data', chunk => {
+        data += chunk
+      })
+
+      res.on('end', () => {
+        if (res.statusCode === 200) {
+          resolve(data)
+        } else {
+          reject(new Error(`Failed to fetch file. Status code: ${res.statusCode}`))
+        }
+      })
+    })
+
+    req.on('error', error => {
+      reject(error)
+    })
+
+    req.end()
+  })
+}
+
+/**
+ * Validates the inputs for a repository operation.
+ *
+ * @param {Object}  inputs - The inputs for the repository operation.
+ * @param {string}  inputs.repoOwner - The owner of the repository.
+ * @param {string}  inputs.repoName - The name of the repository.
+ * @param {string}  inputs.filePath - The file path.
+ * @param {string}  [inputs.branch] - The branch name.
+ * @param {string}  [inputs.ref] - The Git reference.
+ * @throws {Error}  If any of the inputs are invalid.
+ */
+function validateInputs({
+  repoOwner,
+  repoName,
+  filePath,
+  branch,
+  ref,
+}) {
+  if (!VALID_SLUG_REGEX.test(repoOwner)) {
+    throw new Error(`Invalid repoOwner "${repoOwner}"`)
+  }
+  if (!VALID_SLUG_REGEX.test(repoName)) {
+    throw new Error(`Invalid repoName "${repoName}"`)
+  }
+  if (!VALID_FILE_REGEX.test(filePath)) {
+    throw new Error(`Invalid filePath "${filePath}"`)
+  }
+  if (branch && !VALID_FILE_REGEX.test(branch)) {
+    throw new Error(`Invalid branch "${branch}"`)
+  }
+  if (ref && !isGitHash(ref)) {
+    throw new Error(`Invalid ref "${ref}"`)
+  }
+}
+
+function resolveAccessToken(accessToken) {
+  if (typeof accessToken === 'string' && accessToken.match(/process\.env\./)) {
+    return process.env[accessToken.replace('process.env.', '')]
+  }
+  return accessToken || process.env.GITHUB_ACCESS_TOKEN
+}
+
+function decode(fileContent) {
+  return Buffer.from(fileContent, 'base64').toString('utf-8')
+}
+
+function isGitHash(str) {
+  // Regular expression to match Git hashes
+  const gitHashRegex = /^[vV]?[0-9a-fA-F]{40}$/
+  return gitHashRegex.test(str)
+}
+
+function parseLineRange(lineRangeString) {
+  const matches = lineRangeString.match(/#L(\d+)-L(\d+)/)
+  if (!matches) return
+  const startLine = parseInt(matches[1])
+  const endLine = parseInt(matches[2])
+  return [startLine, endLine]
+}
+
+module.exports = {
+  isGithubLink,
+  isGithubRawLink,
+  getGitHubFileContentsRaw,
+  resolveGithubDetails,
+  resolveGithubContents,
+}

package/lib/transforms/code/resolve-github-file.test.js

@@ -0,0 +1,32 @@
+const { resolveGithubContents, getGitHubFileContentsRaw } = require('./resolve-github-file')
+
+let repoFilePath
+repoFilePath = 'https://github.com/DavidWells/markdown-magic/blob/master/package.json'
+// repoFilePath = 'https://github.com/DavidWells/notes/blob/master/cognito.md'
+// repoFilePath = 'github.com/DavidWells/notes/blob/master/cognito.md#L1-L5'
+repoFilePath = 'github.com/DavidWells/notes/blob/master/cognito.md'
+// repoFilePath = 'https://raw.githubusercontent.com/DavidWells/notes/master/cognito.md'
+// repoFilePath = 'raw.githubusercontent.com/DavidWells/notes/master/cognito.md'
+// repoFilePath = 'https://github.com/reapit/foundations/blob/53b2be65ea69d5f1338dbea6e5028c7599d78cf7/packages/connect-session/src/browser/index.ts#L125-L163'
+
+/*
+resolveGithubContents({
+  repoFilePath,
+  debug: true,
+  //accessToken: process.env.GITHUB_LAST_EDITED_TOKEN
+})
+  .then(console.log)
+  .catch(console.error);
+/** */
+
+/*
+getGitHubFileContentsRaw({
+  repoOwner: 'DavidWells', 
+  repoName: 'notes',
+  filePath: 'cognito.md', 
+  branch: 'master', 
+  accessToken: process.env.GITHUB_LAST_EDITED_TOKEN
+})
+  .then(console.log)
+  .catch(console.error);
+/** */

package/lib/utils/fs.js

@@ -2,7 +2,7 @@ const fs = require('fs').promises
 const path = require('path')
 const globrex = require('globrex')
 const isGlob = require('is-glob')
-const isLocalPath = require('is-local-path')
+const _isLocalPath = require('is-local-path')
 const { REGEX_REGEX, escapeRegexString } = require('./regex') 
 const { dirname, resolve, join } = require('path')
 const { readdir, stat, readFile } = fs
@@ -165,6 +165,11 @@ function depth(string) {
   return path.normalize(string).split(path.sep).length - 1;
 }
 
+function isLocalPath(filePath) {
+  if (filePath.startsWith('github.com/') || filePath.startsWith('raw.githubusercontent.com/')) return false
+  return _isLocalPath(filePath)
+}
+
 module.exports = {
   isLocalPath,
   writeFile,

package/lib/utils/remoteRequest.js

@@ -2,13 +2,14 @@ const request = require('sync-request')
 
 module.exports = function remoteRequest(url) {
   let body
+  const finalUrl = (url.match(/^https?:\/\//)) ? url : `https://${url}`
   try {
     // @ts-expect-error
-    const res = request('GET', url)
+    const res = request('GET', finalUrl)
     body = res.getBody('utf8')
   } catch (e) {
-    console.log(`WARNING: REMOTE URL ${url} NOT FOUND`)
-    console.log(e.message)
+    console.log(`WARNING: REMOTE URL ${finalUrl} NOT FOUND`)
+    console.log((e.message || '').split('\n')[0])
   }
   return body
 }

package/lib/utils/text.js

@@ -60,6 +60,14 @@ function replaceTextBetweenChars(str = '', start, end, newStr) {
   return str.substring(0, start) + newStr + str.substring(end)
 }
 
+/**
+ * Retrieves the text content between the specified start and end lines.
+ *
+ * @param {string} content - The content to extract text from.
+ * @param {number} startLine - The line number where the extraction should start.
+ * @param {number} endLine - The line number where the extraction should end.
+ * @returns {string|undefined} - The extracted text content, or undefined if both startLine and endLine are not defined.
+ */
 function getTextBetweenLines(content, startLine, endLine) {
   const startDefined = typeof startLine !== 'undefined'
   const endDefined = typeof endLine !== 'undefined'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "markdown-magic",
-  "version": "3.0.7",
+  "version": "3.0.9",
   "description": "Automatically update markdown files with content from external sources",
   "main": "lib/index.js",
   "bin": {