typst 0.0.1

data/ext/typst/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "typst"
+version = "0.0.1"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+chrono = { version = "0.4.24", default-features = false, features = [
+    "clock",
+    "std",
+] }
+codespan-reporting = "0.11"
+comemo = "0.3"
+dirs = "5"
+ecow = "0.2"
+env_logger = "0.10.1"
+env_proxy = "0.4"
+filetime = "0.2.22"
+flate2 = "1"
+fontdb = "0.15.0"
+log = "0.4.20"
+magnus = { version = "0.6" }
+pathdiff = "0.2"
+same-file = "1"
+siphasher = "1.0"
+tar = "0.4"
+typst = { git = "https://github.com/typst/typst.git", tag = "v0.9.0" }
+typst-library = { git = "https://github.com/typst/typst.git", tag = "v0.9.0" }
+ureq = { version = "2", default-features = false, features = [
+    "gzip",
+    "socks-proxy",
+] }
+walkdir = "2.4.0"
+
+# enable rb-sys feature to test against Ruby head. This is only needed if you
+# want to work with the unreleased, in-development, next version of Ruby
+rb-sys = { version = "*", default-features = false, features = ["stable-api-compiled-fallback"] }

data/ext/typst/extconf.rb

@@ -0,0 +1,4 @@
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("typst/typst")

data/ext/typst/src/compiler.rs

@@ -0,0 +1,187 @@
+use chrono::{Datelike, Timelike};
+use codespan_reporting::diagnostic::{Diagnostic, Label};
+use codespan_reporting::term::{self, termcolor};
+use typst::diag::{Severity, SourceDiagnostic, StrResult};
+use typst::doc::Document;
+use typst::eval::{eco_format, Datetime, Tracer};
+use typst::syntax::{FileId, Source, Span};
+use typst::{World, WorldExt};
+
+use crate::world::SystemWorld;
+
+type CodespanResult<T> = Result<T, CodespanError>;
+type CodespanError = codespan_reporting::files::Error;
+
+impl SystemWorld {
+    pub fn compile_pdf(&mut self) -> StrResult<Vec<u8>> {
+        // Reset everything and ensure that the main file is present.
+        self.reset();
+        self.source(self.main()).map_err(|err| err.to_string())?;
+
+        let mut tracer = Tracer::default();
+        let result = typst::compile(self, &mut tracer);
+        let warnings = tracer.warnings();
+
+        match result {
+            // Export the PDF / PNG.
+            Ok(document) => Ok(export_pdf(&document, self)?),
+            Err(errors) => Err(format_diagnostics(self, &errors, &warnings).unwrap().into()),
+        }
+    }
+
+    pub fn compile_svg(&mut self) -> StrResult<Vec<String>> {
+        // Reset everything and ensure that the main file is present.
+        self.reset();
+        self.source(self.main()).map_err(|err| err.to_string())?;
+
+        let mut tracer = Tracer::default();
+        let result = typst::compile(self, &mut tracer);
+        let warnings = tracer.warnings();
+
+        match result {
+            // Export the PDF / PNG.
+            Ok(document) => Ok(export_svg(&document)?),
+            Err(errors) => Err(format_diagnostics(self, &errors, &warnings).unwrap().into()),
+        }
+    }
+}
+
+/// Export to a PDF.
+#[inline]
+fn export_pdf(document: &Document, world: &SystemWorld) -> StrResult<Vec<u8>> {
+    let ident = world.input().to_string_lossy();
+    let buffer = typst::export::pdf(document, Some(&ident), now());
+    Ok(buffer)
+}
+
+/// Export to one or multiple SVGs.
+#[inline]
+fn export_svg(document: &Document) -> StrResult<Vec<String>> {
+    let mut buffer: Vec<String> = Vec::new();
+    for (_i, frame) in document.pages.iter().enumerate() {
+        let svg = typst::export::svg(frame);
+        buffer.push(svg.to_string())
+    }
+    Ok(buffer)
+}
+
+/// Get the current date and time in UTC.
+fn now() -> Option<Datetime> {
+    let now = chrono::Local::now().naive_utc();
+    Datetime::from_ymd_hms(
+        now.year(),
+        now.month().try_into().ok()?,
+        now.day().try_into().ok()?,
+        now.hour().try_into().ok()?,
+        now.minute().try_into().ok()?,
+        now.second().try_into().ok()?,
+    )
+}
+
+/// Format diagnostic messages.\
+pub fn format_diagnostics(
+    world: &SystemWorld,
+    errors: &[SourceDiagnostic],
+    warnings: &[SourceDiagnostic],
+) -> Result<String, codespan_reporting::files::Error> {
+    let mut w = termcolor::Buffer::no_color();
+
+    let config = term::Config {
+        tab_width: 2,
+        ..Default::default()
+    };
+
+    for diagnostic in warnings.iter().chain(errors.iter()) {
+        let diag = match diagnostic.severity {
+            Severity::Error => Diagnostic::error(),
+            Severity::Warning => Diagnostic::warning(),
+        }
+        .with_message(diagnostic.message.clone())
+        .with_notes(
+            diagnostic
+                .hints
+                .iter()
+                .map(|e| (eco_format!("hint: {e}")).into())
+                .collect(),
+        )
+        .with_labels(label(world, diagnostic.span).into_iter().collect());
+
+        term::emit(&mut w, &config, world, &diag)?;
+
+        // Stacktrace-like helper diagnostics.
+        for point in &diagnostic.trace {
+            let message = point.v.to_string();
+            let help = Diagnostic::help()
+                .with_message(message)
+                .with_labels(label(world, point.span).into_iter().collect());
+
+            term::emit(&mut w, &config, world, &help)?;
+        }
+    }
+
+    let s = String::from_utf8(w.into_inner()).unwrap();
+    Ok(s)
+}
+
+/// Create a label for a span.
+fn label(world: &SystemWorld, span: Span) -> Option<Label<FileId>> {
+    Some(Label::primary(span.id()?, world.range(span)?))
+}
+
+impl<'a> codespan_reporting::files::Files<'a> for SystemWorld {
+    type FileId = FileId;
+    type Name = String;
+    type Source = Source;
+
+    fn name(&'a self, id: FileId) -> CodespanResult<Self::Name> {
+        let vpath = id.vpath();
+        Ok(if let Some(package) = id.package() {
+            format!("{package}{}", vpath.as_rooted_path().display())
+        } else {
+            // Try to express the path relative to the working directory.
+            vpath
+                .resolve(self.root())
+                .and_then(|abs| pathdiff::diff_paths(&abs, self.workdir()))
+                .as_deref()
+                .unwrap_or_else(|| vpath.as_rootless_path())
+                .to_string_lossy()
+                .into()
+        })
+    }
+
+    fn source(&'a self, id: FileId) -> CodespanResult<Self::Source> {
+        Ok(self.lookup(id))
+    }
+
+    fn line_index(&'a self, id: FileId, given: usize) -> CodespanResult<usize> {
+        let source = self.lookup(id);
+        source
+            .byte_to_line(given)
+            .ok_or_else(|| CodespanError::IndexTooLarge {
+                given,
+                max: source.len_bytes(),
+            })
+    }
+
+    fn line_range(&'a self, id: FileId, given: usize) -> CodespanResult<std::ops::Range<usize>> {
+        let source = self.lookup(id);
+        source
+            .line_to_range(given)
+            .ok_or_else(|| CodespanError::LineTooLarge {
+                given,
+                max: source.len_lines(),
+            })
+    }
+
+    fn column_number(&'a self, id: FileId, _: usize, given: usize) -> CodespanResult<usize> {
+        let source = self.lookup(id);
+        source.byte_to_column(given).ok_or_else(|| {
+            let max = source.len_bytes();
+            if given <= max {
+                CodespanError::InvalidCharBoundary { given }
+            } else {
+                CodespanError::IndexTooLarge { given, max }
+            }
+        })
+    }
+}

data/ext/typst/src/download.rs

@@ -0,0 +1,79 @@
+use std::io::{self, ErrorKind, Read};
+
+use ureq::Response;
+
+/// Download binary data and display its progress.
+#[allow(clippy::result_large_err)]
+pub fn download(url: &str) -> Result<Vec<u8>, ureq::Error> {
+    let mut builder =
+        ureq::AgentBuilder::new().user_agent(concat!("typst/{}", env!("CARGO_PKG_VERSION")));
+
+    // Get the network proxy config from the environment.
+    if let Some(proxy) = env_proxy::for_url_str(url)
+        .to_url()
+        .and_then(|url| ureq::Proxy::new(url).ok())
+    {
+        builder = builder.proxy(proxy);
+    }
+
+    let agent = builder.build();
+    let response = agent.get(url).call()?;
+    Ok(RemoteReader::from_response(response).download()?)
+}
+
+/// A wrapper around [`ureq::Response`] that reads the response body in chunks
+/// over a websocket and displays statistics about its progress.
+///
+/// Downloads will _never_ fail due to statistics failing to print, print errors
+/// are silently ignored.
+struct RemoteReader {
+    reader: Box<dyn Read + Send + Sync + 'static>,
+    content_len: Option<usize>,
+}
+
+impl RemoteReader {
+    /// Wraps a [`ureq::Response`] and prepares it for downloading.
+    ///
+    /// The 'Content-Length' header is used as a size hint for read
+    /// optimization, if present.
+    pub fn from_response(response: Response) -> Self {
+        let content_len: Option<usize> = response
+            .header("Content-Length")
+            .and_then(|header| header.parse().ok());
+
+        Self {
+            reader: response.into_reader(),
+            content_len,
+        }
+    }
+
+    /// Download the bodies content as raw bytes while attempting to print
+    /// download statistics to standard error. Download progress gets displayed
+    /// and updated every second.
+    ///
+    /// These statistics will never prevent a download from completing, errors
+    /// are silently ignored.
+    pub fn download(mut self) -> io::Result<Vec<u8>> {
+        let mut buffer = vec![0; 8192];
+        let mut data = match self.content_len {
+            Some(content_len) => Vec::with_capacity(content_len),
+            None => Vec::with_capacity(8192),
+        };
+
+        loop {
+            let read = match self.reader.read(&mut buffer) {
+                Ok(0) => break,
+                Ok(n) => n,
+                // If the data is not yet ready but will be available eventually
+                // keep trying until we either get an actual error, receive data
+                // or an Ok(0).
+                Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
+                Err(e) => return Err(e),
+            };
+
+            data.extend(&buffer[..read]);
+        }
+
+        Ok(data)
+    }
+}

data/ext/typst/src/fonts.rs

@@ -0,0 +1,86 @@
+use std::cell::OnceCell;
+use std::fs::{self};
+use std::path::PathBuf;
+
+use fontdb::{Database, Source};
+use typst::font::{Font, FontBook, FontInfo};
+
+/// Searches for fonts.
+pub struct FontSearcher {
+    // Metadata about all discovered fonts.
+    pub book: FontBook,
+    /// Slots that the fonts are loaded into.
+    pub fonts: Vec<FontSlot>,
+}
+
+/// Holds details about the location of a font and lazily the font itself.
+pub struct FontSlot {
+    /// The path at which the font can be found on the system.
+    path: PathBuf,
+    /// The index of the font in its collection. Zero if the path does not point
+    /// to a collection.
+    index: u32,
+    /// The lazily loaded font.
+    font: OnceCell<Option<Font>>,
+}
+
+impl FontSlot {
+    /// Get the font for this slot.
+    pub fn get(&self) -> Option<Font> {
+        self.font
+            .get_or_init(|| {
+                let data = fs::read(&self.path).ok()?.into();
+                Font::new(data, self.index)
+            })
+            .clone()
+    }
+}
+
+impl FontSearcher {
+    /// Create a new, empty system searcher.
+    pub fn new() -> Self {
+        Self {
+            book: FontBook::new(),
+            fonts: vec![],
+        }
+    }
+
+    /// Search everything that is available.
+    pub fn search(&mut self, font_dirs: &[PathBuf], font_files: &[PathBuf]) {
+        let mut db = Database::new();
+
+        // Font paths have highest priority.
+        for path in font_dirs {
+            db.load_fonts_dir(path);
+        }
+
+        // System fonts have second priority.
+        db.load_system_fonts();
+
+        for path in font_files {
+            let _ret = db.load_font_file(path).ok();
+        }
+
+        for face in db.faces() {
+            let path = match &face.source {
+                Source::File(path) | Source::SharedFile(path, _) => path,
+                // We never add binary sources to the database, so there
+                // shouln't be any.
+                Source::Binary(_) => continue,
+            };
+
+            let info = db
+                .with_face_data(face.id, FontInfo::new)
+                .expect("database must contain this font");
+
+            if let Some(info) = info {
+                self.book.push(info);
+                self.fonts.push(FontSlot {
+                    path: path.clone(),
+                    index: face.index,
+                    font: OnceCell::new(),
+                });
+            }
+        }
+    }
+}

data/ext/typst/src/lib.rs

@@ -0,0 +1,133 @@
+use std::path::PathBuf;
+
+use magnus::{define_module, function, exception, Error, IntoValue};
+use magnus::{prelude::*};
+
+use world::SystemWorld;
+
+mod compiler;
+mod download;
+mod fonts;
+mod package;
+mod world;
+
+fn to_svg(
+    input: PathBuf,
+    root: Option<PathBuf>,
+    font_paths: Vec<PathBuf>,
+    resource_path: PathBuf,
+) -> Result<Vec<String>, Error> {
+    let input = input.canonicalize()
+        .map_err(|err| magnus::Error::new(exception::arg_error(), err.to_string()))?;
+
+    let root = if let Some(root) = root {
+        root.canonicalize()
+            .map_err(|err| magnus::Error::new(exception::arg_error(), err.to_string()))?
+    } else if let Some(dir) = input.parent() {
+        dir.into()
+    } else {
+        PathBuf::new()
+    };
+
+    let resource_path = resource_path.canonicalize()
+        .map_err(|err| magnus::Error::new(exception::arg_error(), err.to_string()))?;
+
+    let mut default_fonts = Vec::new();
+    for entry in walkdir::WalkDir::new(resource_path.join("fonts")) {
+        let path = entry
+            .map_err(|err| magnus::Error::new(exception::arg_error(), err.to_string()))?
+            .into_path();
+        let Some(extension) = path.extension() else {
+            continue;
+        };
+        if extension == "ttf" || extension == "otf" {
+            default_fonts.push(path);
+        }
+    }
+
+    let mut world = SystemWorld::builder(root, input)
+        .font_paths(font_paths)
+        .font_files(default_fonts)
+        .build()
+        .map_err(|msg| magnus::Error::new(exception::arg_error(), msg.to_string()))?;
+
+    let svg_bytes = world
+        .compile_svg()
+        .map_err(|msg| magnus::Error::new(exception::arg_error(), msg.to_string()))?;
+
+    Ok(svg_bytes)
+}
+
+fn to_pdf(
+    input: PathBuf,
+    root: Option<PathBuf>,
+    font_paths: Vec<PathBuf>,
+    resource_path: PathBuf,
+) -> Result<Vec<u8>, Error> {
+    let input = input.canonicalize()
+        .map_err(|err| magnus::Error::new(exception::arg_error(), err.to_string()))?;
+
+    let root = if let Some(root) = root {
+        root.canonicalize()
+            .map_err(|err| magnus::Error::new(exception::arg_error(), err.to_string()))?
+    } else if let Some(dir) = input.parent() {
+        dir.into()
+    } else {
+        PathBuf::new()
+    };
+
+    let resource_path = resource_path.canonicalize()
+        .map_err(|err| magnus::Error::new(exception::arg_error(), err.to_string()))?;
+
+    let mut default_fonts = Vec::new();
+    for entry in walkdir::WalkDir::new(resource_path.join("fonts")) {
+        let path = entry
+            .map_err(|err| magnus::Error::new(exception::arg_error(), err.to_string()))?
+            .into_path();
+        let Some(extension) = path.extension() else {
+            continue;
+        };
+        if extension == "ttf" || extension == "otf" {
+            default_fonts.push(path);
+        }
+    }
+
+    let mut world = SystemWorld::builder(root, input)
+        .font_paths(font_paths)
+        .font_files(default_fonts)
+        .build()
+        .map_err(|msg| magnus::Error::new(exception::arg_error(), msg.to_string()))?;
+
+    let pdf_bytes = world
+        .compile_pdf()
+        .map_err(|msg| magnus::Error::new(exception::arg_error(), msg.to_string()))?;
+
+    Ok(pdf_bytes)
+}
+
+fn write_pdf(
+    input: PathBuf,
+    output: PathBuf,
+    root: Option<PathBuf>,
+    font_paths: Vec<PathBuf>,
+    resource_path: PathBuf,
+) -> Result<magnus::Value, Error> {
+    let pdf_bytes = to_pdf(input, root, font_paths, resource_path)?;
+
+    std::fs::write(output, pdf_bytes)
+        .map_err(|_| magnus::Error::new(exception::arg_error(), "error"))?;
+
+    let value = true.into_value();
+    Ok(value)
+}    
+
+#[magnus::init]
+fn init() -> Result<(), Error> {
+    env_logger::init();
+
+    let module = define_module("Typst")?;
+    module.define_singleton_method("_to_pdf", function!(to_pdf, 4))?;
+    module.define_singleton_method("_write_pdf", function!(write_pdf, 5))?;
+    module.define_singleton_method("_to_svg", function!(to_svg, 4))?;
+    Ok(())
+}

data/ext/typst/src/package.rs

@@ -0,0 +1,64 @@
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use ecow::eco_format;
+use typst::diag::{PackageError, PackageResult};
+use typst::syntax::PackageSpec;
+
+use crate::download::download;
+
+/// Make a package available in the on-disk cache.
+pub fn prepare_package(spec: &PackageSpec) -> PackageResult<PathBuf> {
+    let subdir = format!(
+        "typst/packages/{}/{}/{}",
+        spec.namespace, spec.name, spec.version
+    );
+
+    if let Some(data_dir) = dirs::data_dir() {
+        let dir = data_dir.join(&subdir);
+        if dir.exists() {
+            return Ok(dir);
+        }
+    }
+
+    if let Some(cache_dir) = dirs::cache_dir() {
+        let dir = cache_dir.join(&subdir);
+
+        // Download from network if it doesn't exist yet.
+        if spec.namespace == "preview" && !dir.exists() {
+            download_package(spec, &dir)?;
+        }
+
+        if dir.exists() {
+            return Ok(dir);
+        }
+    }
+
+    Err(PackageError::NotFound(spec.clone()))
+}
+
+/// Download a package over the network.
+fn download_package(spec: &PackageSpec, package_dir: &Path) -> PackageResult<()> {
+    // The `@preview` namespace is the only namespace that supports on-demand
+    // fetching.
+    assert_eq!(spec.namespace, "preview");
+
+    let url = format!(
+        "https://packages.typst.org/preview/{}-{}.tar.gz",
+        spec.name, spec.version
+    );
+
+    let data = match download(&url) {
+        Ok(data) => data,
+        Err(ureq::Error::Status(404, _)) => return Err(PackageError::NotFound(spec.clone())),
+        Err(err) => return Err(PackageError::NetworkFailed(Some(eco_format!("{err}")))),
+    };
+
+    let decompressed = flate2::read::GzDecoder::new(data.as_slice());
+    tar::Archive::new(decompressed)
+        .unpack(package_dir)
+        .map_err(|err| {
+            fs::remove_dir_all(package_dir).ok();
+            PackageError::MalformedArchive(Some(eco_format!("{err}")))
+        })
+}