carray 2.0.1 → 3.0.1

data/lib/carray/frame/join.rb

@@ -0,0 +1,248 @@
+# CAFrame join (memo §5, §11.7, §13.3).
+#
+# Join is a thin delegation to landed addressing primitives: the join key
+# yields an address array (locate_addr for a row-preserving left lookup,
+# align_addr for symmetric set alignment), and each column is gathered by
+# +project+ (length-preserving, miss -> UNDEF, Face-lift, read-only safe).
+# The frame layer just distributes that address to every column.
+
+class CAFrame
+  # Default disambiguation for non-key columns present on both sides (§12-C):
+  # left gets "_left", right gets "_right". +_left+/+_right+ read clearer than
+  # pandas' +_x+/+_y+ for the target audience.
+  DEFAULT_JOIN_SUFFIXES = ["_left", "_right"].freeze
+  private_constant :DEFAULT_JOIN_SUFFIXES
+
+  # Join +other+ on a shared key column.
+  #
+  #   how: :left  (default) — keep all left rows; right columns gathered per
+  #                           left row (locate_addr), misses become UNDEF.
+  #   how: :inner/:outer/:right — set-align both key sets (align_addr) and
+  #                           gather both sides; the aligned key values form
+  #                           the +on+ column.
+  #
+  # A non-key column present on both sides collides. The key column (+on+) is
+  # kept once, never suffixed. By default the collision is resolved by suffixing
+  # both sides (+suffixes: ["_left", "_right"]+); pass a 2-element array to pick
+  # meaningful names up front (+["_obs", "_fcst"]+), or +suffixes: false+ to
+  # raise instead. Rename afterward with +rename+ if needed (§12-C).
+  def join(other, on:, how: :left, suffixes: DEFAULT_JOIN_SUFFIXES)
+    on   = on.to_s
+    lkey = self[on]
+    rkey = other[on]
+    plan = join_name_plan(other, on, suffixes)
+
+    case how
+    when :left
+      join_left(other, on, lkey, rkey, plan)
+    when :inner, :outer, :right
+      join_align(other, on, lkey, rkey, how, plan)
+    else
+      raise ArgumentError, "unknown join mode #{how.inspect}"
+    end
+  end
+
+  # As-of (nearest-key) left join for irregular series (memo §11.7). Each
+  # left row is matched to the nearest +other+ row by +on+ via
+  # locate_nearest_addr (the per-row, row-preserving counterpart of
+  # align_nearest_addr); +direction:+ follows CArray (:floor = most recent
+  # at-or-before, :ceil = next, :round = nearest), and rows with no match
+  # in range or beyond +tolerance:+ come back UNDEF. Same wiring as the left
+  # join, only the address primitive differs.
+  def join_asof(other, on:, direction: :floor, tolerance: nil, suffixes: DEFAULT_JOIN_SUFFIXES)
+    on   = on.to_s
+    plan = join_name_plan(other, on, suffixes)
+    addr = self[on].locate_nearest_addr(other[on], direction: direction, tolerance: tolerance)
+    join_by_addr(other, on, addr, plan)
+  end
+
+  # Conform every variable to an externally supplied reference key set (the
+  # asymmetric sibling of +join+; pandas +reindex+). Given a +reference+ array
+  # of key values, each column is gathered onto it by exact key match
+  # (+locate_addr+): the aligned key column (or index) becomes the reference
+  # itself, and a reference key absent from the source comes back UNDEF in
+  # every other column.
+  #
+  # CAFrame measures no interval and generates nothing -- the caller owns the
+  # reference. This is the primitive for reindexing to a caller-built axis
+  # (e.g. a complete 10-minute time grid): supply that axis as +reference+ and
+  # the gaps fill with UNDEF rows carrying only the reference key.
+  #
+  #   df.align("time", reftime)   # reftime = a CArray of reference key values
+  #
+  # +key+ may be a column name or the index axis name. +reference+ is a CArray
+  # (or Array); its keys are matched against the source key by value, so their
+  # data types must be comparable (a DateTime object key matches by eql?/hash).
+  def align(key, reference)
+    key = key.to_s
+    ref = reference.is_a?(CArray) ? reference : reference.to_ca
+    on_index = !@columns.key?(key) && @axis_name == key && @index
+    addr = ref.locate_addr(on_index ? @index : self[key])
+
+    cols = {}
+    variable_names.each { |name| cols[name] = project_rows(self[name], addr) }
+    if on_index
+      CAFrame.new(cols, axis_name: key, index: ref)
+    else
+      cols[key] = ref
+      CAFrame.new(cols, axis_name: @axis_name,
+                        index: @index && project_rows(@index, addr))
+    end
+  end
+
+  # Paste +other+'s variables beside this frame's, matched by **row position**
+  # (memo §12-C) — a keyless column merge (the column-direction counterpart of
+  # the row-stacking +concat+, and the positional counterpart of the key-aligned
+  # +join+; named after the UNIX +paste+). Both frames must have the same
+  # +nrow+; rows are assumed to already correspond (no key alignment, consistent
+  # with the no-implicit-align stance). Colliding column names are disambiguated
+  # by the same policy as +join+ (default suffix +_left+/+_right+, +suffixes:+ to
+  # override, +suffixes: false+ to raise). This frame's index is kept; +other+'s
+  # index, if any, is not carried (only its columns are pasted). Returns a new
+  # frame.
+  #
+  #   obs.paste(fcst)                          # side by side, same rows
+  #   obs.paste(fcst, suffixes: ["_obs", "_fcst"])
+  def paste(other, suffixes: DEFAULT_JOIN_SUFFIXES)
+    CAFrame.paste(self, other, suffixes: suffixes)
+  end
+
+  # Paste N frames side by side by row position (symmetric N-ary sibling of
+  # the instance +#paste+; see the instance verb's docstring for the pair
+  # semantics).  All frames must have the same +nrow+.  The result's
+  # +axis_name+ and +index+ come from the first frame (peers, but a base is
+  # needed for the index).  Column names that appear in more than one frame
+  # are colliding — pass +suffixes:+ as an Array of exactly K strings (one
+  # per input frame) to disambiguate them; otherwise the collision raises.
+  #
+  #   CAFrame.paste(obs, fcst)
+  #   CAFrame.paste(obs, ecmwf, gfs, suffixes: ["_obs", "_ecmwf", "_gfs"])
+  #   CAFrame.paste([obs, fcst])                              # Array is accepted too
+  def self.paste(*frames, suffixes: DEFAULT_JOIN_SUFFIXES)
+    frames = frames.flatten
+    raise ArgumentError, "paste requires at least one frame" if frames.empty?
+    unless frames.all? { |f| f.is_a?(CAFrame) }
+      raise ArgumentError, "paste expects CAFrame arguments"
+    end
+    first = frames.first
+    nrow  = first.nrow
+    frames.each_with_index do |f, i|
+      next if i.zero?
+      unless f.nrow == nrow
+        raise ArgumentError,
+              "paste: row-count mismatch (frame #{i}: #{f.nrow} vs #{nrow}); " \
+              "use join for key alignment"
+      end
+    end
+
+    # Detect columns that appear in 2+ frames (collisions across the K inputs).
+    name_count = Hash.new(0)
+    frames.each { |f| f.variable_names.each { |name| name_count[name] += 1 } }
+    collisions = name_count.select { |_, n| n > 1 }.keys
+
+    if collisions.any?
+      if suffixes == false
+        raise ArgumentError,
+              "paste column name collision: #{collisions.inspect} " \
+              "(pass suffixes: [\"_a\", \"_b\", ...] with one entry per frame, " \
+              "or rename first)"
+      end
+      # For the 2-frame case DEFAULT_JOIN_SUFFIXES ("_left", "_right") works
+      # as-is; for K > 2 the caller must pass a K-length Array (or use the
+      # pair-only default indirectly via the instance method).
+      unless suffixes.is_a?(Array) && suffixes.size == frames.size &&
+             suffixes.all? { |s| s.is_a?(String) }
+        raise ArgumentError,
+              "paste: suffixes must be an Array of #{frames.size} Strings " \
+              "(one per frame), got #{suffixes.inspect}"
+      end
+    end
+
+    cols = {}
+    frames.each_with_index do |f, i|
+      f.variable_names.each do |name|
+        key = collisions.include?(name) ? "#{name}#{suffixes[i]}" : name
+        if cols.key?(key)
+          raise ArgumentError,
+                "paste: column name #{key.inspect} produced twice " \
+                "(collision after suffix from frame #{i})"
+        end
+        cols[key] = f[name]
+      end
+    end
+    new(cols, axis_name: first.axis_name, index: first.index)
+  end
+
+  private def join_name_plan(other, on, suffixes)
+    collisions = (variable_names & other.variable_names) - [on]
+    return { collisions: [], lsuf: nil, rsuf: nil } if collisions.empty?
+
+    if suffixes == false
+      raise ArgumentError,
+            "join column name collision: #{collisions.inspect} " \
+            "(pass suffixes: [\"_l\", \"_r\"] to disambiguate, or rename first)"
+    end
+    unless suffixes.is_a?(Array) && suffixes.size == 2 && suffixes.all? { |s| s.is_a?(String) }
+      raise ArgumentError,
+            "suffixes must be false or a 2-element array of Strings, got #{suffixes.inspect}"
+    end
+    { collisions: collisions, lsuf: suffixes[0], rsuf: suffixes[1] }
+  end
+
+  private def join_name(name, suffix, plan)
+    plan[:collisions].include?(name) ? "#{name}#{suffix}" : name
+  end
+
+  private def put_join_column(cols, name, col)
+    if cols.key?(name)
+      raise ArgumentError,
+            "join produces a duplicate column #{name.inspect} after suffixing; " \
+            "rename a column first or pass different suffixes"
+    end
+    cols[name] = col
+  end
+
+  private def join_left(other, on, lkey, rkey, plan)
+    join_by_addr(other, on, lkey.locate_addr(rkey), plan)
+  end
+
+  private def join_by_addr(other, on, addr, plan)
+    cols = {}
+    variable_names.each { |name| put_join_column(cols, join_name(name, plan[:lsuf], plan), self[name]) }
+    other.variable_names.each do |name|
+      next if name == on
+      put_join_column(cols, join_name(name, plan[:rsuf], plan), project_rows(other[name], addr))
+    end
+    CAFrame.new(cols, axis_name: @axis_name, index: @index)
+  end
+
+  private def join_align(other, on, lkey, rkey, how, plan)
+    common, a_idx, b_idx = CArray.align_addr(lkey, rkey, join: how)
+    cols = {}
+    variable_names.each do |name|
+      if name == on
+        put_join_column(cols, on, common)
+      else
+        put_join_column(cols, join_name(name, plan[:lsuf], plan), project_rows(self[name], a_idx))
+      end
+    end
+    other.variable_names.each do |name|
+      next if name == on
+      put_join_column(cols, join_name(name, plan[:rsuf], plan), project_rows(other[name], b_idx))
+    end
+    # Carry the left index the same way its columns are gathered (by a_idx),
+    # so it stays consistent with the left join; rows with no left match
+    # (outer/right) become UNDEF.
+    new_index = @index && project_rows(@index, a_idx)
+    CAFrame.new(cols, axis_name: @axis_name, index: new_index)
+  end
+
+  private def project_rows(col, addr)
+    return col.project(addr) if col.ndim == 1
+    trailing = col.shape[1..]
+    t = trailing.inject(1, :*)
+    n = addr.shape[0]
+    flat = addr.reshape(n, 1) * t + CArray.int64(t).seq.reshape(1, t)
+    col.project(flat).reshape(n, *trailing)
+  end
+end

data/lib/carray/frame/records.rb

@@ -0,0 +1,99 @@
+# CAFrame record (row-oriented) input (memo §11.2, §11.5).
+#
+# from_records takes an Array of row Hashes -- the shape JSON.parse yields for
+# a JSON array of objects -- and arranges it into columns. Unlike from_csv,
+# the cell values are already typed Ruby objects (Float / Integer / DateTime /
+# String), so a homogeneous column is built at its native leaf type rather than
+# left as strings ("arrange by the value's own type", not string inference --
+# §4.2 stays intact: no date-like string is parsed, mixed columns stay object).
+#
+# An Array-valued cell that is the same length across every record becomes an
+# N-D column (§11.5): { "temp" => [min, mean, max] } over N records is one
+# (N, 3) column. Ragged or non-numeric arrays fall back to an object column.
+#
+# Missing keys and explicit nils become UNDEF for numeric columns (mask, not a
+# float promotion): an int column with a hole stays int + UNDEF.
+
+class CAFrame
+  # Build a frame from an Array of row Hashes. Column set is the union of keys
+  # in first-appearance order; keys are stringified. +types:+ casts named
+  # columns afterward (same map / array-key forms as +cast+).
+  def self.from_records(records, types: nil)
+    unless records.is_a?(Array) && records.all? { |r| r.is_a?(Hash) }
+      raise ArgumentError, "from_records expects an Array of Hashes"
+    end
+    return new if records.empty?
+
+    keys = record_key_union(records)
+    n = records.size
+    cols = {}
+    keys.each do |key|
+      values = records.map { |r| r[key] }
+      cols[key.to_s] = build_record_column(values, n)
+    end
+
+    frame = new(cols)
+    frame.cast(types) if types
+    frame
+  end
+
+  # Union of record keys in first-appearance order (original key objects, so a
+  # String- or Symbol-keyed set both work; the column name stringifies later).
+  def self.record_key_union(records)
+    seen = {}
+    keys = []
+    records.each do |r|
+      r.each_key do |k|
+        unless seen.key?(k)
+          seen[k] = true
+          keys << k
+        end
+      end
+    end
+    keys
+  end
+  private_class_method :record_key_union
+
+  # Arrange one column's values (Ruby objects, nil for missing) into a CArray.
+  # All-Array cells of equal length -> N-D native column; numeric scalars ->
+  # native scalar column (nil -> UNDEF); anything else -> object column.
+  def self.build_record_column(values, n)
+    present = values.reject(&:nil?)
+    return CArray.object(n) { values } if present.empty?
+
+    if present.all? { |v| v.is_a?(Array) || v.is_a?(CArray) }
+      build_nd_column(values, present, n)
+    else
+      type = numeric_leaf_type(present)
+      col = CArray.object(n) { values }
+      type ? col.to_type(type) : col
+    end
+  end
+  private_class_method :build_record_column
+
+  # Stack equal-length array cells into an (N, L) column via an object 2-D fill
+  # + to_type (nil rows -> UNDEF, int/float by leaf). Ragged lengths or
+  # non-numeric leaves fall back to a 1-D object column of the raw cells.
+  def self.build_nd_column(values, present, n)
+    lengths = present.map { |v| v.is_a?(CArray) ? v.shape[0] : v.size }
+    len = lengths.first
+    return CArray.object(n) { values } unless lengths.all? { |x| x == len }
+
+    nested = values.map { |v| v.nil? ? Array.new(len) : (v.is_a?(CArray) ? v.to_a : v) }
+    type = numeric_leaf_type(nested.flatten.compact)
+    table = CArray.object(n, len) { nested }
+    type ? table.to_type(type) : table
+  end
+  private_class_method :build_nd_column
+
+  # :int64 if every value is an Integer, :float64 if all are Numeric (int/float
+  # mix), otherwise nil (strings / DateTime / booleans / mixed -> keep object).
+  def self.numeric_leaf_type(values)
+    if values.all? { |v| v.is_a?(Integer) }
+      :int64
+    elsif values.all? { |v| v.is_a?(Numeric) }
+      :float64
+    end
+  end
+  private_class_method :numeric_leaf_type
+end

data/lib/carray/frame/sort.rb

@@ -0,0 +1,113 @@
+# CAFrame row sort (memo §5, spine = addressing + gather).
+#
+# Two entry points, both delegating to the multi-key CArray.sort_addr -- the
+# class-method lexicographic sort that returns flat row addresses (distinct from
+# the single-key instance CArray#sort_index) -- and gathering every column and
+# the index by that address into a view-frame in key order:
+#
+#   sort_by_key(*keys, order:, masked_position:)  -- sort by named columns, with
+#                           per-key or blanket ascending/descending.
+#   sort_by { |f| ... }  -- sort by key CArrays the block builds (the escape,
+#                           §4.3): derived or composite keys are ordinary column
+#                           math.
+#
+# sort_addr is ascending, so a descending key is expressed by replacing it with
+# its dense descending rank (CArray#order(descending: true, method: :dense)):
+# this works for every data type (unlike negation, which cannot reverse a string and
+# silently wraps an unsigned integer), and the *dense* rank keeps equal values
+# on one rank so ties fall through to later keys in a multi-key sort.
+# masked_position (CArray.sort_addr's kwarg) places masked key rows first or
+# last (default last).
+
+class CAFrame
+  # Sort rows by one or more key columns, lexicographic (the first key is
+  # primary), returning a view-frame. Delegates to +CArray.sort_addr+ for the
+  # row permutation and gathers every column and the index by it.
+  #
+  # Each key is a **column name** (or the index axis name), or a
+  # **`[name, :asc | :desc]`** pair for a per-key direction. The +order:+ keyword
+  # is the direction for bare-name keys (default +:asc+). +masked_position:+
+  # sends masked key rows to the +:last+ (default) or +:first+ end.
+  #
+  #   df.sort_by_key("temp")                              # one column, ascending
+  #   df.sort_by_key("station", "temp")                   # lexicographic
+  #   df.sort_by_key("temp", order: :desc)                # all keys descending
+  #   df.sort_by_key(["station", :asc], ["temp", :desc])  # per-key direction
+  #   df.sort_by_key("temp", masked_position: :first)     # masked rows first
+  #   df.sort_by_key("time")                              # by the index
+  def sort_by_key(*specs, order: :asc, masked_position: :last)
+    raise ArgumentError, "sort_by_key requires at least one key" if specs.empty?
+    validate_sort_order(order)
+    keys = specs.map do |spec|
+      name, dir = parse_sort_key_spec(spec, order)
+      col = sort_key_column(name)
+      dir == :desc ? col.order(descending: true, method: :dense) : col
+    end
+    select_rows(CArray.sort_addr(*keys, masked_position: masked_position))
+  end
+
+  # Sort rows by key columns the block builds (memo §4.3 escape). The block
+  # receives the frame and returns a key CArray, or an Array of them for a
+  # multi-key lexicographic sort; rows are sorted ascending by +CArray.sort_addr+
+  # over those keys and gathered into a view-frame. This is the sort sibling of
+  # +filter+: where +sort_by_key+ takes plain column names, the block form lets
+  # you build any key with CArray ops -- a descending key is its dense
+  # descending rank, derived or composite keys are ordinary column math.
+  # +masked_position:+ places masked key rows first or last (default last).
+  #
+  #   df.sort_by { |f| (f["temp"] - target).abs }   # nearest-to-target first
+  #   df.sort_by { |f| f["temp"].order(descending: true, method: :dense) }
+  def sort_by(masked_position: :last)
+    raise ArgumentError, "sort_by requires a block" unless block_given?
+    keys = yield(self)
+    keys = [keys] unless keys.is_a?(Array)
+    raise ArgumentError, "sort_by block must return a key CArray or an Array of them" if keys.empty?
+    keys.each_with_index do |k, i|
+      unless k.is_a?(CArray)
+        raise ArgumentError, "sort_by key #{i} must be a CArray (got #{k.class})"
+      end
+      unless k.ndim == 1 && k.shape[0] == nrow
+        raise ArgumentError,
+              "sort_by key #{i} must be a 1-D column of length #{nrow} (got shape #{k.shape.inspect})"
+      end
+    end
+    select_rows(CArray.sort_addr(*keys, masked_position: masked_position))
+  end
+
+  private def parse_sort_key_spec(spec, default_order)
+    case spec
+    when String, Symbol
+      [spec, default_order]
+    when Array
+      unless spec.size == 2 && (spec[0].is_a?(String) || spec[0].is_a?(Symbol))
+        raise ArgumentError, "sort key must be a name or [name, :asc|:desc], got #{spec.inspect}"
+      end
+      validate_sort_order(spec[1])
+      spec
+    else
+      raise ArgumentError, "sort key must be a name or [name, :asc|:desc], got #{spec.class}"
+    end
+  end
+
+  private def validate_sort_order(dir)
+    return if dir == :asc || dir == :desc
+    raise ArgumentError, "sort order must be :asc or :desc, got #{dir.inspect}"
+  end
+
+  private def sort_key_column(name)
+    key = name.to_s
+    col = if @columns.key?(key)
+            @columns[key]
+          elsif @index && @axis_name == key
+            @index
+          else
+            raise KeyError, "no column #{key.inspect}"
+          end
+    unless col.ndim == 1
+      raise ArgumentError,
+            "sort_by_key: key column #{key.inspect} is #{col.ndim}-D; " \
+            "sort by a scalar (1-D) column"
+    end
+    col
+  end
+end