prophet-rb 0.3.1 → 0.4.1

data/lib/prophet/diagnostics.rb

@@ -0,0 +1,349 @@
+module Prophet
+  module Diagnostics
+    def self.generate_cutoffs(df, horizon, initial, period)
+      # Last cutoff is 'latest date in data - horizon' date
+      cutoff = df["ds"].max - horizon
+      if cutoff < df["ds"].min
+        raise Error, "Less data than horizon."
+      end
+      result = [cutoff]
+      while result[-1] >= df["ds"].min + initial
+        cutoff -= period
+        # If data does not exist in data range (cutoff, cutoff + horizon]
+        if !(((df["ds"] > cutoff) & (df["ds"] <= cutoff + horizon)).any?)
+          # Next cutoff point is 'last date before cutoff in data - horizon'
+          if cutoff > df["ds"].min
+            closest_date = df[df["ds"] <= cutoff].max["ds"]
+            cutoff = closest_date - horizon
+          end
+          # else no data left, leave cutoff as is, it will be dropped.
+        end
+        result << cutoff
+      end
+      result = result[0...-1]
+      if result.length == 0
+        raise Error, "Less data than horizon after initial window. Make horizon or initial shorter."
+      end
+      # logger.info("Making #{result.length} forecasts with cutoffs between #{result[-1]} and #{result[0]}")
+      result.reverse
+    end
+
+    def self.cross_validation(model, horizon:, period: nil, initial: nil, cutoffs: nil)
+      if model.history.nil?
+        raise Error, "Model has not been fit. Fitting the model provides contextual parameters for cross validation."
+      end
+
+      df = model.history.dup
+      horizon = timedelta(horizon)
+
+      predict_columns = ["ds", "yhat"]
+      if model.uncertainty_samples
+        predict_columns.concat(["yhat_lower", "yhat_upper"])
+      end
+
+      # Identify largest seasonality period
+      period_max = 0.0
+      model.seasonalities.each do |_, s|
+        period_max = [period_max, s[:period]].max
+      end
+      seasonality_dt = timedelta("#{period_max} days")
+
+      if cutoffs.nil?
+        # Set period
+        period = period.nil? ? 0.5 * horizon : timedelta(period)
+
+        # Set initial
+        initial = initial.nil? ? [3 * horizon, seasonality_dt].max : timedelta(initial)
+
+        # Compute Cutoffs
+        cutoffs = generate_cutoffs(df, horizon, initial, period)
+      else
+        # add validation of the cutoff to make sure that the min cutoff is strictly greater than the min date in the history
+        if cutoffs.min <= df["ds"].min
+          raise Error, "Minimum cutoff value is not strictly greater than min date in history"
+        end
+        # max value of cutoffs is <= (end date minus horizon)
+        end_date_minus_horizon = df["ds"].max - horizon
+        if cutoffs.max > end_date_minus_horizon
+          raise Error, "Maximum cutoff value is greater than end date minus horizon, no value for cross-validation remaining"
+        end
+        initial = cutoffs[0] - df["ds"].min
+      end
+
+      # Check if the initial window
+      # (that is, the amount of time between the start of the history and the first cutoff)
+      # is less than the maximum seasonality period
+      if initial < seasonality_dt
+        msg = "Seasonality has period of #{period_max} days "
+        msg += "which is larger than initial window. "
+        msg += "Consider increasing initial."
+        # logger.warn(msg)
+      end
+
+      predicts = cutoffs.map { |cutoff| single_cutoff_forecast(df, model, cutoff, horizon, predict_columns) }
+
+      # Combine all predicted DataFrame into one DataFrame
+      predicts.reduce(Rover::DataFrame.new) { |memo, v| memo.concat(v) }
+    end
+
+    def self.single_cutoff_forecast(df, model, cutoff, horizon, predict_columns)
+      # Generate new object with copying fitting options
+      m = prophet_copy(model, cutoff)
+      # Train model
+      history_c = df[df["ds"] <= cutoff]
+      if history_c.shape[0] < 2
+        raise Error, "Less than two datapoints before cutoff. Increase initial window."
+      end
+      m.fit(history_c, **model.fit_kwargs)
+      # Calculate yhat
+      index_predicted = (df["ds"] > cutoff) & (df["ds"] <= cutoff + horizon)
+      # Get the columns for the future dataframe
+      columns = ["ds"]
+      if m.growth == "logistic"
+        columns << "cap"
+        if m.logistic_floor
+          columns << "floor"
+        end
+      end
+      columns.concat(m.extra_regressors.keys)
+      columns.concat(m.seasonalities.map { |_, props| props[:condition_name] }.compact)
+      yhat = m.predict(df[index_predicted][columns])
+      # Merge yhat(predicts), y(df, original data) and cutoff
+      yhat[predict_columns].merge(df[index_predicted][["y"]]).merge(Rover::DataFrame.new({"cutoff" => [cutoff] * yhat.length}))
+    end
+
+    def self.prophet_copy(m, cutoff = nil)
+      if m.history.nil?
+        raise Error, "This is for copying a fitted Prophet object."
+      end
+
+      if m.specified_changepoints
+        changepoints = m.changepoints
+        if !cutoff.nil?
+          # Filter change points '< cutoff'
+          last_history_date = m.history["ds"][m.history["ds"] <= cutoff].max
+          changepoints = changepoints[changepoints < last_history_date]
+        end
+      else
+        changepoints = nil
+      end
+
+      # Auto seasonalities are set to False because they are already set in
+      # m.seasonalities.
+      m2 = m.class.new(
+        growth: m.growth,
+        n_changepoints: m.n_changepoints,
+        changepoint_range: m.changepoint_range,
+        changepoints: changepoints,
+        yearly_seasonality: false,
+        weekly_seasonality: false,
+        daily_seasonality: false,
+        holidays: m.holidays,
+        seasonality_mode: m.seasonality_mode,
+        seasonality_prior_scale: m.seasonality_prior_scale,
+        changepoint_prior_scale: m.changepoint_prior_scale,
+        holidays_prior_scale: m.holidays_prior_scale,
+        mcmc_samples: m.mcmc_samples,
+        interval_width: m.interval_width,
+        uncertainty_samples: m.uncertainty_samples
+      )
+      m2.extra_regressors = deepcopy(m.extra_regressors)
+      m2.seasonalities = deepcopy(m.seasonalities)
+      m2.country_holidays = deepcopy(m.country_holidays)
+      m2
+    end
+
+    def self.timedelta(value)
+      if value.is_a?(Numeric)
+        # ActiveSupport::Duration is a numeric
+        value
+      elsif (m = /\A(\d+(\.\d+)?) days\z/.match(value))
+        m[1].to_f * 86400
+      else
+        raise Error, "Unknown time delta"
+      end
+    end
+
+    def self.deepcopy(value)
+      if value.is_a?(Hash)
+        value.to_h { |k, v| [deepcopy(k), deepcopy(v)] }
+      elsif value.is_a?(Array)
+        value.map { |v| deepcopy(v) }
+      else
+        value.dup
+      end
+    end
+
+    def self.performance_metrics(df, metrics: nil, rolling_window: 0.1, monthly: false)
+      valid_metrics = ["mse", "rmse", "mae", "mape", "mdape", "smape", "coverage"]
+      if metrics.nil?
+        metrics = valid_metrics
+      end
+      if (df["yhat_lower"].nil? || df["yhat_upper"].nil?) && metrics.include?("coverage")
+        metrics.delete("coverage")
+      end
+      if metrics.uniq.length != metrics.length
+        raise ArgumentError, "Input metrics must be a list of unique values"
+      end
+      if !Set.new(metrics).subset?(Set.new(valid_metrics))
+        raise ArgumentError, "Valid values for metrics are: #{valid_metrics}"
+      end
+      df_m = df.dup
+      if monthly
+        raise Error, "Not implemented yet"
+        # df_m["horizon"] = df_m["ds"].dt.to_period("M").astype(int) - df_m["cutoff"].dt.to_period("M").astype(int)
+      else
+        df_m["horizon"] = df_m["ds"] - df_m["cutoff"]
+      end
+      df_m.sort_by! { |r| r["horizon"] }
+      if metrics.include?("mape") && df_m["y"].abs.min < 1e-8
+        # logger.info("Skipping MAPE because y close to 0")
+        metrics.delete("mape")
+      end
+      if metrics.length == 0
+        return nil
+      end
+      w = (rolling_window * df_m.shape[0]).to_i
+      if w >= 0
+        w = [w, 1].max
+        w = [w, df_m.shape[0]].min
+      end
+      # Compute all metrics
+      dfs = {}
+      metrics.each do |metric|
+        dfs[metric] = send(metric, df_m, w)
+      end
+      res = dfs[metrics[0]]
+      metrics.each do |metric|
+        res_m = dfs[metric]
+        res[metric] = res_m[metric]
+      end
+      res
+    end
+
+    def self.rolling_mean_by_h(x, h, w, name)
+      # Aggregate over h
+      df = Rover::DataFrame.new({"x" => x, "h" => h})
+      df2 = df.group("h").sum("x").inner_join(df.group("h").count).sort_by { |r| r["h"] }
+      xs = df2["sum_x"]
+      ns = df2["count"]
+      hs = df2["h"]
+
+      trailing_i = df2.length - 1
+      x_sum = 0
+      n_sum = 0
+      # We don't know output size but it is bounded by len(df2)
+      res_x = [nil] * df2.length
+
+      # Start from the right and work backwards
+      (df2.length - 1).downto(0) do |i|
+        x_sum += xs[i]
+        n_sum += ns[i]
+        while n_sum >= w
+          # Include points from the previous horizon. All of them if still
+          # less than w, otherwise weight the mean by the difference
+          excess_n = n_sum - w
+          excess_x = excess_n * xs[i] / ns[i]
+          res_x[trailing_i] = (x_sum - excess_x) / w
+          x_sum -= xs[trailing_i]
+          n_sum -= ns[trailing_i]
+          trailing_i -= 1
+        end
+      end
+
+      res_h = hs[(trailing_i + 1)..-1]
+      res_x = res_x[(trailing_i + 1)..-1]
+
+      Rover::DataFrame.new({"horizon" => res_h, name => res_x})
+    end
+
+    def self.rolling_median_by_h(x, h, w, name)
+      # Aggregate over h
+      df = Rover::DataFrame.new({"x" => x, "h" => h})
+      grouped = df.group("h")
+      df2 = grouped.count.sort_by { |r| r["h"] }
+      hs = df2["h"]
+
+      res_h = []
+      res_x = []
+      # Start from the right and work backwards
+      i = hs.length - 1
+      while i >= 0
+        h_i = hs[i]
+        xs = df[df["h"] == h_i]["x"].to_a
+
+        next_idx_to_add = (h == h_i).to_numo.cast_to(Numo::UInt8).argmax - 1
+        while xs.length < w && next_idx_to_add >= 0
+          # Include points from the previous horizon. All of them if still
+          # less than w, otherwise just enough to get to w.
+          xs << x[next_idx_to_add]
+          next_idx_to_add -= 1
+        end
+        if xs.length < w
+          # Ran out of points before getting enough.
+          break
+        end
+        res_h << hs[i]
+        res_x << Rover::Vector.new(xs).median
+        i -= 1
+      end
+      res_h.reverse!
+      res_x.reverse!
+      Rover::DataFrame.new({"horizon" => res_h, name => res_x})
+    end
+
+    def self.mse(df, w)
+      se = (df["y"] - df["yhat"]) ** 2
+      if w < 0
+        return Rover::DataFrame.new({"horizon" => df["horizon"], "mse" => se})
+      end
+      rolling_mean_by_h(se, df["horizon"], w, "mse")
+    end
+
+    def self.rmse(df, w)
+      res = mse(df, w)
+      res["rmse"] = res.delete("mse").map { |v| Math.sqrt(v) }
+      res
+    end
+
+    def self.mae(df, w)
+      ae = (df["y"] - df["yhat"]).abs
+      if w < 0
+        return Rover::DataFrame.new({"horizon" => df["horizon"], "mae" => ae})
+      end
+      rolling_mean_by_h(ae, df["horizon"], w, "mae")
+    end
+
+    def self.mape(df, w)
+      ape = ((df["y"] - df["yhat"]) / df["y"]).abs
+      if w < 0
+        return Rover::DataFrame.new({"horizon" => df["horizon"], "mape" => ape})
+      end
+      rolling_mean_by_h(ape, df["horizon"], w, "mape")
+    end
+
+    def self.mdape(df, w)
+      ape = ((df["y"] - df["yhat"]) / df["y"]).abs
+      if w < 0
+        return Rover::DataFrame.new({"horizon" => df["horizon"], "mdape" => ape})
+      end
+      rolling_median_by_h(ape, df["horizon"], w, "mdape")
+    end
+
+    def self.smape(df, w)
+      sape = (df["y"] - df["yhat"]).abs / ((df["y"].abs + df["yhat"].abs) / 2)
+      if w < 0
+        return Rover::DataFrame.new({"horizon" => df["horizon"], "smape" => sape})
+      end
+      rolling_mean_by_h(sape, df["horizon"], w, "smape")
+    end
+
+    def self.coverage(df, w)
+      is_covered = (df["y"] >= df["yhat_lower"]) & (df["y"] <= df["yhat_upper"])
+      if w < 0
+        return Rover::DataFrame.new({"horizon" => df["horizon"], "coverage" => is_covered})
+      end
+      rolling_mean_by_h(is_covered.to(:float), df["horizon"], w, "coverage")
+    end
+  end
+end

data/lib/prophet/forecaster.rb

@@ -3,7 +3,14 @@ module Prophet
     include Holidays
     include Plot
 
-    attr_reader :logger, :params, :train_holiday_names
+    attr_reader :logger, :params, :train_holiday_names,
+      :history, :seasonalities, :specified_changepoints, :fit_kwargs,
+      :growth, :changepoints, :n_changepoints, :changepoint_range,
+      :holidays, :seasonality_mode, :seasonality_prior_scale,
+      :holidays_prior_scale, :changepoint_prior_scale, :mcmc_samples,
+      :interval_width, :uncertainty_samples
+
+    attr_accessor :extra_regressors, :seasonalities, :country_holidays
 
     def initialize(
       growth: "linear",
@@ -176,8 +183,10 @@ module Prophet
 
       initialize_scales(initialize_scales, df)
 
-      if @logistic_floor && !df.include?("floor")
-        raise ArgumentError, "Expected column \"floor\"."
+      if @logistic_floor
+        unless df.include?("floor")
+          raise ArgumentError, "Expected column \"floor\"."
+        end
       else
         df["floor"] = 0
       end
@@ -207,7 +216,12 @@ module Prophet
     def initialize_scales(initialize_scales, df)
       return unless initialize_scales
 
-      floor = 0
+      if @growth == "logistic" && df.include?("floor")
+        @logistic_floor = true
+        floor = df["floor"]
+      else
+        floor = 0.0
+      end
       @y_scale = (df["y"] - floor).abs.max
       @y_scale = 1 if @y_scale == 0
       @start = df["ds"].min
@@ -386,6 +400,12 @@ module Prophet
 
     def add_country_holidays(country_name)
       raise Error, "Country holidays must be added prior to model fitting." if @history
+
+      # Fix for previously documented keyword argument
+      if country_name.is_a?(Hash) && country_name[:country_name]
+        country_name = country_name[:country_name]
+      end
+
       # Validate names.
       get_holiday_names(country_name).each do |name|
         # Allow merging with existing holidays
@@ -965,6 +985,12 @@ module Prophet
       Rover::DataFrame.new({"ds" => dates})
     end
 
+    def to_json
+      require "json"
+
+      JSON.generate(as_json)
+    end
+
     private
 
     # Time is preferred over DateTime in Ruby docs
@@ -1011,5 +1037,189 @@ module Prophet
       u = Numo::DFloat.new(size).rand(-0.5, 0.5)
       loc - scale * u.sign * Numo::NMath.log(1 - 2 * u.abs)
     end
+
+    SIMPLE_ATTRIBUTES = [
+      "growth", "n_changepoints", "specified_changepoints", "changepoint_range",
+      "yearly_seasonality", "weekly_seasonality", "daily_seasonality",
+      "seasonality_mode", "seasonality_prior_scale", "changepoint_prior_scale",
+      "holidays_prior_scale", "mcmc_samples", "interval_width", "uncertainty_samples",
+      "y_scale", "logistic_floor", "country_holidays", "component_modes"
+    ]
+
+    PD_SERIES = ["changepoints", "history_dates", "train_holiday_names"]
+
+    PD_TIMESTAMP = ["start"]
+
+    PD_TIMEDELTA = ["t_scale"]
+
+    PD_DATAFRAME = ["holidays", "history", "train_component_cols"]
+
+    NP_ARRAY = ["changepoints_t"]
+
+    ORDEREDDICT = ["seasonalities", "extra_regressors"]
+
+    def as_json
+      if @history.nil?
+        raise Error, "This can only be used to serialize models that have already been fit."
+      end
+
+      model_dict =
+        SIMPLE_ATTRIBUTES.to_h do |attribute|
+          [attribute, instance_variable_get("@#{attribute}")]
+        end
+
+      # Handle attributes of non-core types
+      PD_SERIES.each do |attribute|
+        if instance_variable_get("@#{attribute}").nil?
+          model_dict[attribute] = nil
+        else
+          v = instance_variable_get("@#{attribute}")
+          d = {
+            "name" => "ds",
+            "index" => v.size.times.to_a,
+            "data" => v.to_a.map { |v| v.iso8601(3) }
+          }
+          model_dict[attribute] = JSON.generate(d)
+        end
+      end
+      PD_TIMESTAMP.each do |attribute|
+        model_dict[attribute] = instance_variable_get("@#{attribute}").to_f
+      end
+      PD_TIMEDELTA.each do |attribute|
+        model_dict[attribute] = instance_variable_get("@#{attribute}").to_f
+      end
+      PD_DATAFRAME.each do |attribute|
+        if instance_variable_get("@#{attribute}").nil?
+          model_dict[attribute] = nil
+        else
+          # use same format as Pandas
+          v = instance_variable_get("@#{attribute}")
+
+          v = v.dup
+          v["ds"] = v["ds"].map { |v| v.iso8601(3) } if v["ds"]
+          v.delete("col")
+
+          fields =
+            v.types.map do |k, t|
+              type =
+                case t
+                when :object
+                  "datetime"
+                when :int64
+                  "integer"
+                else
+                  "number"
+                end
+              {"name" => k, "type" => type}
+            end
+
+          d = {
+            "schema" => {
+              "fields" => fields,
+              "pandas_version" => "0.20.0"
+            },
+            "data" => v.to_a
+          }
+          model_dict[attribute] = JSON.generate(d)
+        end
+      end
+      NP_ARRAY.each do |attribute|
+        model_dict[attribute] = instance_variable_get("@#{attribute}").to_a
+      end
+      ORDEREDDICT.each do |attribute|
+        model_dict[attribute] = [
+          instance_variable_get("@#{attribute}").keys,
+          instance_variable_get("@#{attribute}").transform_keys(&:to_s)
+        ]
+      end
+      # Other attributes with special handling
+      # fit_kwargs -> Transform any numpy types before serializing.
+      # They do not need to be transformed back on deserializing.
+      # TODO deep copy
+      fit_kwargs = @fit_kwargs.to_h { |k, v| [k.to_s, v.dup] }
+      if fit_kwargs.key?("init")
+        fit_kwargs["init"].each do |k, v|
+          if v.is_a?(Numo::NArray)
+            fit_kwargs["init"][k] = v.to_a
+          # elsif v.is_a?(Float)
+          #   fit_kwargs["init"][k] = v.to_f
+          end
+        end
+      end
+      model_dict["fit_kwargs"] = fit_kwargs
+
+      # Params (Dict[str, np.ndarray])
+      model_dict["params"] = params.transform_values(&:to_a)
+      # Attributes that are skipped: stan_fit, stan_backend
+      # Returns 1.0 for Prophet 1.1
+      model_dict["__prophet_version"] = "1.0"
+      model_dict
+    end
+
+    def self.from_json(model_json)
+      require "json"
+
+      model_dict = JSON.parse(model_json)
+
+      # We will overwrite all attributes set in init anyway
+      model = Prophet.new
+      # Simple types
+      SIMPLE_ATTRIBUTES.each do |attribute|
+        model.instance_variable_set("@#{attribute}", model_dict.fetch(attribute))
+      end
+      PD_SERIES.each do |attribute|
+        if model_dict[attribute].nil?
+          model.instance_variable_set("@#{attribute}", nil)
+        else
+          d = JSON.parse(model_dict.fetch(attribute))
+          s = Rover::Vector.new(d["data"])
+          if d["name"] == "ds"
+            s = s.map { |v| Time.parse(v).utc }
+          end
+          model.instance_variable_set("@#{attribute}", s)
+        end
+      end
+      PD_TIMESTAMP.each do |attribute|
+        model.instance_variable_set("@#{attribute}", Time.at(model_dict.fetch(attribute)))
+      end
+      PD_TIMEDELTA.each do |attribute|
+        model.instance_variable_set("@#{attribute}", model_dict.fetch(attribute).to_f)
+      end
+      PD_DATAFRAME.each do |attribute|
+        if model_dict[attribute].nil?
+          model.instance_variable_set("@#{attribute}", nil)
+        else
+          d = JSON.parse(model_dict.fetch(attribute))
+          df = Rover::DataFrame.new(d["data"])
+          df["ds"] = df["ds"].map { |v| Time.parse(v).utc } if df["ds"]
+          if attribute == "train_component_cols"
+            # Special handling because of named index column
+            # df.columns.name = 'component'
+            # df.index.name = 'col'
+          end
+          model.instance_variable_set("@#{attribute}", df)
+        end
+      end
+      NP_ARRAY.each do |attribute|
+        model.instance_variable_set("@#{attribute}", Numo::NArray.cast(model_dict.fetch(attribute)))
+      end
+      ORDEREDDICT.each do |attribute|
+        key_list, unordered_dict = model_dict.fetch(attribute)
+        od = {}
+        key_list.each do |key|
+          od[key] = unordered_dict[key].transform_keys(&:to_sym)
+        end
+        model.instance_variable_set("@#{attribute}", od)
+      end
+      # Other attributes with special handling
+      # fit_kwargs
+      model.instance_variable_set(:@fit_kwargs, model_dict["fit_kwargs"].transform_keys(&:to_sym))
+      # Params (Dict[str, np.ndarray])
+      model.instance_variable_set(:@params, model_dict["params"].transform_values { |v| Numo::NArray.cast(v) })
+      # Skipped attributes
+      # model.stan_backend = nil
+      model.instance_variable_set(:@stan_fit, nil)
+      model
+    end
   end
 end

data/lib/prophet/holidays.rb

@@ -2,25 +2,21 @@ module Prophet
   module Holidays
     def get_holiday_names(country)
       years = (1995..2045).to_a
-      make_holidays_df(years, country)["holiday"].uniq
+      holiday_names = make_holidays_df(years, country)["holiday"].uniq
+      # TODO raise error in 0.4.0
+      logger.warn "Holidays in #{country} are not currently supported"
+      holiday_names
     end
 
     def make_holidays_df(year_list, country)
       holidays_df[(holidays_df["country"] == country) & (holidays_df["year"].in?(year_list))][["ds", "holiday"]]
     end
 
-    # TODO marshal on installation
+    # TODO improve performance
     def holidays_df
       @holidays_df ||= begin
-        holidays = {"ds" => [], "holiday" => [], "country" => [], "year" => []}
         holidays_file = File.expand_path("../../data-raw/generated_holidays.csv", __dir__)
-        CSV.foreach(holidays_file, headers: true, converters: [:date, :numeric]) do |row|
-          holidays["ds"] << row["ds"]
-          holidays["holiday"] << row["holiday"]
-          holidays["country"] << row["country"]
-          holidays["year"] << row["year"]
-        end
-        Rover::DataFrame.new(holidays)
+        Rover.read_csv(holidays_file, converters: [:date, :numeric])
       end
     end
   end