mpp_reader 0.1.0

data/lib/mpp_reader/cli.rb

@@ -0,0 +1,110 @@
+require "optparse"
+require "json"
+
+module MppReader
+  class CLI
+    def self.run(argv, stdout: $stdout, stderr: $stderr)
+      options = { json: false }
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: mpp_reader FILE... [options]"
+        opts.on("--json", "Print one JSON object per file to stdout instead of a task tree") do
+          options[:json] = true
+        end
+        opts.on("--version", "Print version") do
+          stdout.puts MppReader::VERSION
+          return 0
+        end
+      end
+
+      begin
+        files = parser.parse(argv)
+      rescue OptionParser::ParseError => e
+        stderr.puts e.message
+        stderr.puts parser.banner
+        return 2
+      end
+      if files.empty?
+        stderr.puts parser.banner
+        return 2
+      end
+
+      status = 0
+      files.each do |file|
+        project = MppReader.open(file)
+        if options[:json]
+          stdout.puts JSON.generate(json_for(project, file))
+        else
+          print_tree(project, file, stdout)
+        end
+      rescue MppReader::Error, SystemCallError => e
+        stderr.puts "#{file}: #{e.message}"
+        status = 1
+      end
+      status
+    end
+
+    def self.print_tree(project, file, stdout)
+      stdout.puts "#{file} (#{project.file_format}, #{project.application_name})"
+      stdout.puts "\nTasks (#{project.tasks.size}):"
+      project.tasks.each do |t|
+        indent = "  " * (t.outline_level || 0)
+        dates = [t.start&.strftime("%Y-%m-%d"), t.finish&.strftime("%Y-%m-%d")].compact.join(" .. ")
+        extras = []
+        extras << "#{t.duration.value.round(2)} #{t.duration.units}" if t.duration
+        extras << "#{t.percent_complete}%" if t.percent_complete&.positive?
+        extras << "milestone" if t.milestone?
+        extras << "inactive" unless t.active
+        line = "#{indent}[#{t.unique_id}] #{t.name} #{dates} #{extras.join(', ')}".rstrip
+        stdout.puts line
+        stdout.puts "#{indent}    > #{t.notes.gsub("\n", "\n#{indent}    > ")}" if t.notes
+      end
+      stdout.puts "\nResources (#{project.resources.size}):"
+      project.resources.each { |r| stdout.puts "  [#{r.unique_id}] #{r.name}" }
+    end
+
+    def self.json_for(project, file)
+      {
+        file: file,
+        format: project.file_format,
+        application: project.application_name,
+        tasks: project.tasks.map do |t|
+          {
+            unique_id: t.unique_id, id: t.id, name: t.name,
+            outline_level: t.outline_level,
+            start: iso(t.start), finish: iso(t.finish),
+            duration: duration(t.duration),
+            percent_complete: t.percent_complete,
+            milestone: t.milestone?, summary: t.summary?,
+            active: t.active, manually_scheduled: t.manual,
+            parent_unique_id: t.parent&.unique_id,
+            predecessors: t.predecessors.map do |r|
+              { predecessor_task_unique_id: r.predecessor_task_unique_id,
+                type: r.type, lag: duration(r.lag) }
+            end,
+            notes: t.notes
+          }.compact
+        end,
+        resources: project.resources.map do |r|
+          { unique_id: r.unique_id, id: r.id, name: r.name,
+            calendar_unique_id: r.calendar_unique_id, notes: r.notes }.compact
+        end,
+        assignments: project.assignments.map do |a|
+          { unique_id: a.unique_id, task_unique_id: a.task_unique_id,
+            resource_unique_id: a.resource_unique_id,
+            start: iso(a.start), finish: iso(a.finish),
+            units: a.units, work: duration(a.work), notes: a.notes }.compact
+        end,
+        calendars: project.calendars.map do |c|
+          { unique_id: c.unique_id, name: c.name,
+            base_calendar_unique_id: c.base_calendar_unique_id,
+            resource_unique_id: c.resource_unique_id,
+            days: c.days, exceptions: c.exceptions }.compact
+        end
+      }
+    end
+
+    def self.iso(time) = time&.strftime("%Y-%m-%dT%H:%M:%S")
+
+    def self.duration(d) = d && { value: d.value, units: d.units }
+  end
+end

data/lib/mpp_reader/comp_obj.rb

@@ -0,0 +1,37 @@
+module MppReader
+  # The \x01CompObj stream identifies the application that wrote an OLE
+  # compound file and its file format. Layout (ported from MPXJ CompObj):
+  # 28-byte header, then length-prefixed NUL-terminated ANSI strings for
+  # application name, file format and application ID. "Microsoft Project 4.0"
+  # files carry only the application name.
+  class CompObj
+    attr_reader :application_name, :file_format, :application_id
+
+    def initialize(data)
+      @pos = 28
+      @data = data
+      @application_name = read_string
+      if @application_name == "Microsoft Project 4.0"
+        @file_format = "MSProject.MPP4"
+        @application_id = "MSProject.Project.4"
+      else
+        @file_format = read_string
+        @application_id = read_string
+      end
+    end
+
+    private
+
+    def read_string
+      return nil if @pos + 4 > @data.bytesize
+
+      length = @data.byteslice(@pos, 4).unpack1("V")
+      @pos += 4
+      return nil if length <= 0 || @pos + length > @data.bytesize
+
+      str = @data.byteslice(@pos, length - 1) # drop trailing NUL
+      @pos += length
+      str
+    end
+  end
+end

data/lib/mpp_reader/decode.rb

@@ -0,0 +1,105 @@
+require "date"
+
+module MppReader
+  # A duration as MS Project models it: a magnitude in scheduling units
+  # (working time), not an absolute time span.
+  Duration = Struct.new(:value, :units)
+
+  # Decoders for MS Project's binary value encodings, ported from MPXJ
+  # MPPUtility. The epoch is 1983-12-31; dates are u16 days since the epoch;
+  # times of day and durations are stored in tenths of a minute; timestamps
+  # combine a u16 time (6-second units) with a u16 day count.
+  module Decode
+    EPOCH_DATE = Date.new(1983, 12, 31)
+
+    DURATION_UNITS_MASK = 0x1F
+    DURATION_UNITS = {
+      3 => :minutes,
+      4 => :elapsed_minutes,
+      5 => :hours,
+      6 => :elapsed_hours,
+      7 => :days,
+      8 => :elapsed_days,
+      9 => :weeks,
+      10 => :elapsed_weeks,
+      11 => :months,
+      12 => :elapsed_months,
+      19 => :percent,
+      20 => :elapsed_percent
+    }.freeze
+
+    # Divisors converting tenths-of-minutes to each unit, assuming MS
+    # Project's defaults (8h days, 40h weeks, 20-day months).
+    TENTHS_PER_UNIT = {
+      minutes: 10, elapsed_minutes: 10,
+      hours: 600, elapsed_hours: 600,
+      days: 4800, elapsed_days: 14_400,
+      weeks: 24_000, elapsed_weeks: 100_800,
+      months: 96_000, elapsed_months: 432_000
+    }.freeze
+
+    module_function
+
+    def date(data, offset)
+      days = data.byteslice(offset, 2).to_s.unpack1("v")
+      return nil if days.nil? || days == 0xFFFF
+
+      EPOCH_DATE + days
+    end
+
+    def timestamp(data, offset)
+      days = data.byteslice(offset + 2, 2).to_s.unpack1("v")
+      return nil if days.nil? || days <= 1 || days == 0xFFFF
+
+      time = data.byteslice(offset, 2).unpack1("v")
+      time = 0 if time == 0xFFFF
+      seconds = time * 6
+      # Very small day counts show as NA in MS Project; a non-zero seconds
+      # component distinguishes NA from real values (MPXJ heuristic).
+      return nil if days < 100 && (seconds % 60) != 0
+
+      to_time(EPOCH_DATE + days, seconds)
+    end
+
+    def timestamp_from_tenths(data, offset)
+      tenths = data.byteslice(offset, 4).to_s.unpack1("V")
+      return nil if tenths.nil?
+
+      seconds = tenths * 6
+      to_time(EPOCH_DATE + (seconds / 86_400), seconds % 86_400)
+    end
+
+    def duration_units(type, default: :days)
+      code = type & DURATION_UNITS_MASK
+      return default if code == 21
+
+      DURATION_UNITS.fetch(code, :days)
+    end
+
+    # value is in tenths of a minute; converts using MS Project's default
+    # hours-per-day assumptions.
+    def duration(value, units)
+      Duration.new(value.to_f / TENTHS_PER_UNIT.fetch(units, 1), units)
+    end
+
+    # Like duration, but day/week/month conversions honour the project's
+    # configured working time instead of the defaults.
+    def adjusted_duration(value, units, minutes_per_day:, minutes_per_week:, days_per_month:)
+      tenths_per_unit =
+        case units
+        when :days then minutes_per_day * 10
+        when :weeks then minutes_per_week * 10
+        when :months then minutes_per_day * days_per_month * 10
+        end
+      if tenths_per_unit.nil? || tenths_per_unit.zero?
+        duration(value, units)
+      else
+        Duration.new(value.to_f / tenths_per_unit, units)
+      end
+    end
+
+    def to_time(date, seconds_of_day)
+      Time.new(date.year, date.month, date.day) + seconds_of_day
+    end
+  end
+end

data/lib/mpp_reader/errors.rb

@@ -0,0 +1,13 @@
+module MppReader
+  class Error < StandardError; end
+
+  # Not an OLE file, or an OLE file that is not an MS Project file.
+  class InvalidFormatError < Error; end
+
+  # A recognized MS Project file in a format this gem does not support
+  # (e.g. MPP8/MPP9/MPP12, or a password-protected file).
+  class UnsupportedFormatError < Error; end
+
+  # Structurally broken file (bad FAT chains, truncated streams, bad records).
+  class CorruptFileError < Error; end
+end

data/lib/mpp_reader/field_map.rb

@@ -0,0 +1,106 @@
+require_relative "field_tables"
+
+module MppReader
+  # Maps logical fields (task name, start, ...) to where their values live in
+  # an entity's data blocks. The definition is stored in the project Props
+  # under entity-specific keys, as 28-byte entries (ported from MPXJ
+  # FieldMap.createFieldMap): mask u32@0, fixed-data offset u16@4, field type
+  # u32@12, category u16@20.
+  #
+  # Locations: :fixed_data (offset into a FixedData record; a second fixed
+  # block is signalled by offsets restarting), :var_data (key into Var2Data;
+  # for MPP14 the key is the low word of the field type), :meta_data (bit
+  # mask over a FixedMeta record).
+  class FieldMap
+    Item = Struct.new(:location, :data_block, :fixed_offset, :var_key, :mask,
+                      :meta_block, :data_type, :units_field)
+
+    # Bytes each data type occupies in a fixed-data record (MPXJ
+    # FieldMap#getFixedDataFieldSize); used to size-check records.
+    FIXED_DATA_SIZES = Hash.new(0).update(
+      date: 4, integer: 4, duration: 4,
+      time_units: 2, constraint: 2, priority: 2, percentage: 2, task_type: 2,
+      accrue: 2, short: 2, boolean: 2, delay: 2, workgroup: 2, rate_units: 2,
+      earned_value_method: 2, resource_request_type: 2,
+      currency: 8, units: 8, rate: 8, work: 8,
+      work_units: 1, guid: 16
+    ).freeze
+
+    ENTITIES = {
+      task: [FieldTables::TASK_FIELD_BASE, FieldTables::TASK_FIELDS,
+             FieldTables::TASK_FIELD_TYPES, [131_092, 50_331_668]],
+      resource: [FieldTables::RESOURCE_FIELD_BASE, FieldTables::RESOURCE_FIELDS,
+                 FieldTables::RESOURCE_FIELD_TYPES, [131_093, 50_331_669]],
+      assignment: [FieldTables::ASSIGNMENT_FIELD_BASE, FieldTables::ASSIGNMENT_FIELDS,
+                   FieldTables::ASSIGNMENT_FIELD_TYPES, [131_095, 50_331_671]]
+    }.freeze
+
+    def self.for_tasks(props) = from_props(props, :task)
+    def self.for_resources(props) = from_props(props, :resource)
+    def self.for_assignments(props) = from_props(props, :assignment)
+
+    def self.from_props(props, entity)
+      data = ENTITIES.fetch(entity)[3].filter_map { |key| props[key] }.first
+      if data.nil?
+        raise UnsupportedFormatError,
+              "no #{entity} field map in project Props - default field maps are not implemented"
+      end
+
+      new(data, entity: entity)
+    end
+
+    def initialize(data, entity:)
+      base, fields, types, = ENTITIES.fetch(entity)
+      @map = {}
+      @max_fixed_data_size = Hash.new(0)
+      last_offset = 0
+      data_block = 0
+      pos = 0
+
+      while pos + 28 <= data.bytesize
+        mask = data.byteslice(pos, 4).unpack1("V")
+        fixed_offset = data.byteslice(pos + 4, 2).unpack1("v")
+        type_value = data.byteslice(pos + 12, 4).unpack1("V")
+        category = data.byteslice(pos + 20, 2).unpack1("v")
+        pos += 28
+
+        next unless (type_value & 0xFFFF0000) == base
+
+        field = fields[type_value & 0xFFFF]
+        next if field.nil?
+
+        # NOTE: MPXJ substitutes var data keys for some custom fields
+        # (FieldMap14.VAR_DATA_MAP); not implemented yet.
+        var_key = type_value & 0xFFFF
+
+        item =
+          case category
+          when 0x0B then Item.new(:meta_data, 0, nil, nil, mask, 0)
+          when 0x64 then Item.new(:meta_data, 0, nil, nil, mask, 1)
+          else
+            if fixed_offset != 0xFFFF
+              data_block += 1 if fixed_offset < last_offset
+              last_offset = fixed_offset
+              Item.new(:fixed_data, data_block, fixed_offset, nil, mask, 0)
+            elsif var_key != 0
+              Item.new(:var_data, 0, nil, var_key, mask, 0)
+            end
+          end
+        next if item.nil?
+
+        item.data_type, item.units_field = types[field]
+        if item.location == :fixed_data
+          field_end = item.fixed_offset + FIXED_DATA_SIZES[item.data_type]
+          if field_end > @max_fixed_data_size[item.data_block]
+            @max_fixed_data_size[item.data_block] = field_end
+          end
+        end
+        @map[field] = item
+      end
+    end
+
+    def [](field) = @map[field]
+
+    def max_fixed_data_size(block) = @max_fixed_data_size[block]
+  end
+end

data/lib/mpp_reader/field_reader.rb

@@ -0,0 +1,119 @@
+module MppReader
+  # Decodes a single field value given its FieldMap::Item, the entity's
+  # fixed-data records and the var data block. Ported from MPXJ
+  # FieldMap.FieldItem#read; only the data types needed so far are
+  # implemented - unknown types decode to nil rather than raising.
+  class FieldReader
+    MINUTES_PER_DAY_KEY = 37_748_765
+    MINUTES_PER_WEEK_KEY = 37_748_766
+    DAYS_PER_MONTH_KEY = 37_753_743
+
+    def initialize(props)
+      @minutes_per_day = positive_or(props.int(MINUTES_PER_DAY_KEY), 480)
+      @minutes_per_week = positive_or(props.int(MINUTES_PER_WEEK_KEY), 2400)
+      @days_per_month = positive_or(props.short(DAYS_PER_MONTH_KEY), 20)
+    end
+
+    # fixed_records is an array of byte strings, one per fixed-data block
+    # (block 0 = FixedData, block 1 = Fixed2Data).
+    def read(field_map, field, unique_id, fixed_records, var_data)
+      item = field_map[field]
+      return nil if item.nil?
+
+      case item.location
+      when :fixed_data
+        record = fixed_records[item.data_block]
+        return nil if record.nil? || item.fixed_offset >= record.bytesize
+
+        read_fixed(field_map, item, record, unique_id, fixed_records, var_data)
+      when :var_data
+        read_var(field_map, item, unique_id, fixed_records, var_data)
+      end
+    end
+
+    # Converts a raw duration value (tenths of a minute) using the
+    # project's configured working time.
+    def adjusted_duration(value, units)
+      Decode.adjusted_duration(value, units,
+                               minutes_per_day: @minutes_per_day,
+                               minutes_per_week: @minutes_per_week,
+                               days_per_month: @days_per_month)
+    end
+
+    private
+
+    def positive_or(value, default)
+      value && value.positive? ? value : default
+    end
+
+    def read_fixed(field_map, item, record, unique_id, fixed_records, var_data)
+      offset = item.fixed_offset
+      case item.data_type
+      when :date
+        Decode.timestamp(record, offset)
+      when :integer
+        record.byteslice(offset, 4).to_s.unpack1("l<")
+      when :duration
+        units = duration_units_of(field_map, item, unique_id, fixed_records, var_data)
+        value = record.byteslice(offset, 4).to_s.unpack1("l<")
+        value.nil? || value == -1 ? nil : adjusted_duration(value, units)
+      when :time_units
+        Decode.duration_units(u16(record, offset).to_i)
+      when :percentage
+        value = u16(record, offset)
+        value && value <= 100 ? value : nil
+      when :short, :constraint, :priority, :task_type
+        u16(record, offset)
+      when :boolean
+        u16(record, offset).to_i != 0
+      when :currency, :units
+        double = record.byteslice(offset, 8).to_s.unpack1("E")
+        double && double.abs >= 0.1 ? double / 100 : 0.0
+      when :work
+        double = record.byteslice(offset, 8).to_s.unpack1("E")
+        Duration.new(double && double.abs >= 1000 ? double / 60_000 : 0.0, :hours)
+      end
+    end
+
+    def read_var(field_map, item, unique_id, fixed_records, var_data)
+      key = item.var_key
+      case item.data_type
+      when :string
+        var_data.string(unique_id, key)
+      when :notes
+        RtfText.strip(var_data.byte_string(unique_id, key))
+      when :date
+        bytes = var_data.bytes(unique_id, key)
+        bytes && Decode.timestamp(bytes, 0)
+      when :integer
+        var_data.bytes(unique_id, key)&.unpack1("l<")
+      when :numeric
+        var_data.bytes(unique_id, key)&.unpack1("E")
+      when :currency
+        double = var_data.bytes(unique_id, key)&.unpack1("E")
+        double && double.abs >= 0.1 ? double / 100 : nil
+      when :work
+        double = var_data.bytes(unique_id, key)&.unpack1("E")
+        double && Duration.new(double.abs < 1000 ? 0.0 : double / 60_000, :hours)
+      when :duration
+        units = duration_units_of(field_map, item, unique_id, fixed_records, var_data)
+        value = var_data.bytes(unique_id, key)&.unpack1("l<")
+        value && adjusted_duration(value, units)
+      when :percentage, :short
+        var_data.bytes(unique_id, key)&.unpack1("v")
+      when :boolean
+        var_data.bytes(unique_id, key)&.unpack1("v").to_i != 0
+      end
+    end
+
+    def duration_units_of(field_map, item, unique_id, fixed_records, var_data)
+      units = item.units_field &&
+              read(field_map, item.units_field, unique_id, fixed_records, var_data)
+      units.is_a?(Symbol) ? units : :days
+    end
+
+    def u16(record, offset)
+      record.byteslice(offset, 2).to_s.unpack1("v")
+    end
+  end
+end