postjob 0.5.11 → 0.5.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e26be7454c3a0ddb3d26d67b4e51918e8e16e306
-  data.tar.gz: f837746f80bba5cebbdd0fea7c7ad7c26eacbc5b
+  metadata.gz: 175e2a30ec560a1728f3b3a62750bd5afb38df3d
+  data.tar.gz: 54328faf7c7966c21743575ab8d7a9c94b66180d
 SHA512:
-  metadata.gz: 361995efdeda5d1ca51de93c0807e37737581c32f53c25f20dc45c50c65dc008260abccc0d421776d59186997071fd2aa14cc23828f95145de6536bcf7859589
-  data.tar.gz: f374453e87325c9ddb05d4b1cad18f20d26b3a6c64a596de23ab471bc6bbc57017b20df5c589944e1cc90994ad717e4d1e57b7fa8ec6ab953ef48e6ec0a33adc
+  metadata.gz: 98960758ed5262988f82ac8cb046752be425b15bd91b2355e575f69007d43b7161580d2796af4d4bd6ebcea71d87f955d442f061ad43ba720efb02a2da172584
+  data.tar.gz: 5f1bdd2b015c706cddc7e5874bcbfada7ebc4d3c22f34511d6f5d14eb0c3f28261eebd834670cbd1e68920776b1132a20482d7c0bc58e4a13d55edbb175ee996

data/lib/postjob/cli/cron.rb

@@ -1,4 +1,28 @@
+# rubocop:disable Lint/HandleExceptions
+
 module Postjob::CLI
+  # Lists cron jobs in the system
+  def cron(limit: "30", queue: nil)
+    limit = Integer(limit)
+    queue = queue.split(",") if queue
+
+    connect_to_database!
+
+    query = ps_query(limit: limit, queue: queue, tags: nil)
+    query = query.where("cron_interval IS NOT NULL")
+    # query = query
+    print_results query: query
+  end
+
+  def cron_top(limit: "30", queue: nil)
+    loop do
+      system "clear"
+      cron limit: limit, queue: queue
+      sleep 1
+    end
+  rescue Interrupt
+  end
+
   # Enqueues a cron workflow
   def cron_enqueue(workflow, *args, interval:, queue: "ruby", tags: nil)
     interval = Integer(interval)

data/lib/postjob/cli/db.rb

@@ -8,8 +8,7 @@ module Postjob::CLI
 
     raise "No settings in this database schema" unless ::Postjob::Queue.settings?
 
-    records = "SELECT name,value FROM postjob.settings ORDER BY name"
-    tp Simple::SQL.all(records, into: Hash)
+    Simple::SQL.print "SELECT name,value FROM postjob.settings ORDER BY name"
   end
 
   def db_migrate

data/lib/postjob/cli/events.rb

@@ -37,7 +37,7 @@ module Postjob::CLI
   # Example:
   #
   #   postjob events
-  def events(limit: "100")
+  def events(limit: "30")
     expect! limit => /\A\d+\z/
     limit = Integer(limit)
 
@@ -49,7 +49,7 @@ module Postjob::CLI
   end
 
   # Show up-to-date events information once per second
-  def events_top(limit: "100")
+  def events_top(limit: "30")
     loop do
       system "clear"
       events(limit: limit)

data/lib/postjob/cli/heartbeat.rb

@@ -31,7 +31,7 @@ module Postjob::CLI
   public
 
   # Show the latest heartbeat events
-  def heartbeat(limit: "100")
+  def heartbeat(limit: "30")
     expect! limit => /\A\d+\z/
     limit = Integer(limit)
 
@@ -44,7 +44,7 @@ module Postjob::CLI
   end
 
   # Show up-to-date heartbeat information once per second
-  def heartbeat_top(limit: "100")
+  def heartbeat_top(limit: "30")
     loop do
       system "clear"
       heartbeat(limit: limit)

data/lib/postjob/cli/helpers.rb

@@ -0,0 +1,28 @@
+
+
+module Postjob::CLI
+  private
+
+  def parse_ids(*ids)
+    ids.map do |s|
+      s = s.gsub(/.*\./, "")
+      Integer(s)
+    end.uniq
+  end
+
+  def print_results(query:, on_empty: nil, title: nil)
+    if title
+      puts "\n=== #{title} =======================================\n\n"
+    end
+
+    records = Simple::SQL.print(query, into: Hash)
+
+    if records.total_count && records.total_count > records.length
+      logger.warn "Output limited up to limit #{records.length}. Use the --limit=<NN> command line option for a different limit."
+    end
+
+    if records.empty? && on_empty
+      logger.warn(on_empty)
+    end
+  end
+end

data/lib/postjob/cli/hosts.rb

@@ -10,21 +10,21 @@ module Postjob::CLI
     sql = <<-SQL
       SELECT
         hosts.id,
-        hosts.attributes,
         hosts.created_at,
-        heartbeat.attributes AS heartbeat,
-        heartbeat.created_at AS heartbeat_created_at
+        hosts.status,
+        heartbeats.attributes AS heartbeat,
+        heartbeats.created_at AS heartbeat_created_at,
+        hosts.attributes
       FROM postjob.hosts hosts
       LEFT JOIN (
         SELECT
-          worker_sessions.host_id,
-          MAX(events.id) AS event_id
-        FROM postjob.worker_sessions
-        LEFT JOIN postjob.events events ON events.worker_session_id=worker_sessions.id
-        WHERE events.name = 'heartbeat'
-        GROUP BY worker_sessions.host_id
-      ) q ON q.host_id=hosts.id
-      LEFT JOIN events heartbeat ON heartbeat.id=event_id
+          host_id,
+          attributes,
+          created_at,
+          rank() OVER (PARTITION BY host_id ORDER BY created_at DESC) AS rank
+        FROM postjob.events events
+        WHERE name='heartbeat'
+      ) heartbeats ON heartbeats.host_id=hosts.id AND rank = 1
     SQL
 
     scope = Simple::SQL::Scope.new(sql)
@@ -42,7 +42,7 @@ module Postjob::CLI
   # Example:
   #
   #   postjob hosts
-  def hosts(limit: "100")
+  def hosts(limit: "30")
     expect! limit => /\A\d+\z/
     limit = Integer(limit)
 
@@ -54,9 +54,7 @@ module Postjob::CLI
   end
 
   # Show up-to-date hosts information once per second
-  #
-  #
-  def hosts_top(limit: "100")
+  def hosts_top(limit: "30")
     loop do
       system "clear"
       hosts(limit: limit)
@@ -64,4 +62,23 @@ module Postjob::CLI
     end
   rescue Interrupt
   end
+
+  # resets the host id
+  def host_reset
+    Postjob::Host.clear_storage
+  end
+
+  # Set the host to shutdown state
+  def host_shutdown
+    connect_to_database!
+
+    Simple::SQL.ask "UPDATE postjob.hosts SET status='shutdown' WHERE id=$1::uuid", ::Postjob.host_id
+  end
+
+  # Set the host to running again
+  def host_restart
+    connect_to_database!
+
+    Simple::SQL.ask "UPDATE postjob.hosts SET status='running' WHERE id=$1::uuid", ::Postjob.host_id
+  end
 end

data/lib/postjob/cli/job.rb

@@ -87,6 +87,8 @@ module Postjob::CLI
   public
 
   def registry
+    require "table_print"
+
     workflows_with_versions = Postjob::Registry.workflows.keys.reject { |k| k[1] == "" }
     workflows_with_versions = workflows_with_versions.sort_by { |name, _version| name }
 

data/lib/postjob/cli/ps.rb

@@ -29,7 +29,7 @@ module Postjob::CLI
         max_attempts,
         cron_interval AS cron,
         CASE
-          WHEN is_sticky THEN COALESCE(substring(sticky_host_id::varchar for 6) || '...', 'yes')
+          WHEN is_sticky THEN COALESCE(substring(sticky_host_id::varchar for 9) || '...', 'yes')
           ELSE 'no'
           END AS sticky,
         is_greedy AS greedy,
@@ -64,7 +64,7 @@ module Postjob::CLI
   #
   # For a listing of all jobs in the system use ps:full, see 'postjob help ps:full'
   # for details.
-  def ps(*ids, limit: "100", tags: nil, queue: nil, only_root: nil)
+  def ps(*ids, limit: "30", tags: nil, queue: nil, only_root: nil)
     expect! limit => /\A\d+\z/
     limit = Integer(limit)
     queue = queue.split(",") if queue
@@ -85,7 +85,7 @@ module Postjob::CLI
     print_results query: query
   end
 
-  def ps_full(*ids, limit: "100", tags: nil, queue: nil)
+  def ps_full(*ids, limit: "30", tags: nil, queue: nil)
     queue = queue.split(",") if queue
 
     connect_to_database!
@@ -101,7 +101,7 @@ module Postjob::CLI
   end
 
   # Show up-to-date information once per second
-  def ps_top(*ids, limit: "100", tags: nil, full: false, queue: nil, only_root: false)
+  def ps_top(*ids, limit: "30", tags: nil, full: false, queue: nil, only_root: false)
     loop do
       system "clear"
       if full
@@ -159,26 +159,4 @@ module Postjob::CLI
       pp job
     end
   end
-
-  private
-
-  def parse_ids(*ids)
-    ids.map do |s|
-      s = s.gsub(/.*\./, "")
-      Integer(s)
-    end.uniq
-  end
-
-  def print_results(query:, on_empty: nil)
-    records = Simple::SQL.all(query, into: Hash)
-    tp records
-
-    if records.total_count > records.length
-      logger.warn "Output limited up to limit #{records.length}. Use the --limit=<NN> command line option for a different limit."
-    end
-
-    if records.empty? && on_empty
-      logger.warn(on_empty)
-    end
-  end
 end

data/lib/postjob/cli/queues.rb

@@ -0,0 +1,66 @@
+# rubocop:disable Metrics/MethodLength
+# rubocop:disable Lint/HandleExceptions
+
+module Postjob::CLI
+  private
+
+  def queues_query(time_name:)
+    sql = <<-SQL
+    SELECT
+      queue,
+      status,
+      iff(status = 'ready', waiting, processing) AS "#{time_name}"
+    FROM (
+      SELECT
+        queue,
+        status,
+        COUNT(*),
+        AVG (now() at time zone 'utc' - jobs.created_at) AS waiting,
+        AVG (jobs.updated_at - jobs.created_at) AS processing
+      FROM postjob.postjobs jobs
+      WHERE jobs.root_id=jobs.id
+        AND (jobs.next_run_at IS NULL OR jobs.next_run_at < (now() at time zone 'utc'))
+      GROUP BY queue, status
+      ORDER BY queue, status
+    ) sq
+    SQL
+
+    scope = Simple::SQL::Scope.new(sql)
+    scope
+  end
+
+  public
+
+  # Show hosts status
+  #
+  # This command lists all worker_sessions currently in the system.
+  #
+  # Example:
+  #
+  #   postjob hosts
+  def queues
+    connect_to_database!
+
+    query = queues_query(time_name: "waiting for..")
+    query = query.where(status: %w(ready))
+    print_results query: query, title: "Waiting jobs"
+
+    query = queues_query(time_name: "processing since..")
+    query = query.where(status: %w(sleep processing err))
+    print_results query: query, title: "Processing jobs"
+
+    query = queues_query(time_name: "processing time")
+    query = query.where(status: %w(failed ok timeout))
+    print_results query: query, title: "Finished jobs"
+  end
+
+  # # Show up-to-date hosts information once per second
+  def queues_top
+    loop do
+      system "clear"
+      queues
+      sleep 1
+    end
+  rescue Interrupt
+  end
+end

data/lib/postjob/cli/run.rb

@@ -1,3 +1,6 @@
+# rubocop:disable Metrics/PerceivedComplexity
+# rubocop:disable Metrics/ParameterLists
+
 module Postjob::CLI
   # Run a single job
   def step
@@ -10,21 +13,31 @@ module Postjob::CLI
   #
   # Parameters:
   #
-  # - count=<count> maximum number of jobs to process. Default: unlimited.
-  # - queue run only the specified queue.
-  # - quiet don't show progress.
+  # - --count=<count> maximum number of jobs to process. Default: unlimited.
+  # - --queue=queue1,queue2,queue3 run only the specified queues.
+  # - --heartbeat=no don't start heartbeat process.
+  # - --quiet don't show progress.
   #
-  def run(count: nil, queue: nil, quiet: false, fast: false)
+  def run(count: nil, queue: "ruby", quiet: false, fast: false, host_id: nil, heartbeat: true)
+    expect! Integer(host_id, 16) => 1..0xffffffff if host_id
     count = Integer(count) if count
-    queue = queue.split(",") if queue
+
+    expect! heartbeat => [ "yes", "no" ] if heartbeat.is_a?(String)
+    heartbeat = %w(yes true).include?(heartbeat) if heartbeat.is_a?(String)
+    expect! heartbeat => [ true, false ]
 
     Postjob.fast_mode = (fast ? true : false)
 
     connect_to_database!
 
+    if host_id
+      Postjob::Host.host_id = "%08x-0000-0000-0000-000000000000" % Integer(host_id, 16)
+      Postjob.logger.info "Using host_id: #{Postjob::Host.host_id}"
+    end
+
     logger.success "Starting runner with pid #{$$}"
 
-    processed = Postjob.run(count: count, queue: queue) do |job_id|
+    processed = Postjob.run(count: count, queues: queue.split(","), heartbeat: heartbeat) do |job_id|
       logger.info "Processed job w/id #{job_id}" if job_id
       STDERR.print "." unless quiet
     end

data/lib/postjob/cli/sessions.rb

@@ -10,7 +10,8 @@ module Postjob::CLI
     sql = <<-SQL
       SELECT
         worker_sessions.id,
-        worker_sessions.host_id,
+        (substring(worker_sessions.host_id::varchar for 9) || '...') AS host_id,
+        array_to_string(worker_sessions.queues, ', ') AS queues,
         worker_sessions.client_socket,
         worker_sessions.workflows,
         worker_sessions.created_at,
@@ -42,7 +43,7 @@ module Postjob::CLI
     SQL
 
     scope = Simple::SQL::Scope.new(sql)
-
+    scope = scope.where("worker_sessions.id != '00000000-0000-0000-0000-000000000000'::uuid")
     scope
       .paginate(per: limit, page: 1)
       .order_by("heartbeat_created_at DESC NULLS LAST")
@@ -57,7 +58,7 @@ module Postjob::CLI
   # Example:
   #
   #   postjob sessions
-  def sessions(limit: "100")
+  def sessions(limit: "30")
     expect! limit => /\A\d+\z/
     limit = Integer(limit)
 
@@ -72,7 +73,7 @@ module Postjob::CLI
   end
 
   # Show up-to-date session information once per second
-  def sessions_top(limit: "100")
+  def sessions_top(limit: "30")
     loop do
       system "clear"
       sessions(limit: limit)

data/lib/postjob/host.rb

@@ -2,6 +2,11 @@ require_relative "./record"
 require "tempfile"
 
 class Postjob::Host < Postjob::Record
+  attr_reader :id
+  attr_reader :attributes
+  attr_reader :status
+  attr_reader :created_at
+
   class << self
     def clear_storage
       @host_id = nil
@@ -9,8 +14,19 @@ class Postjob::Host < Postjob::Record
       File.unlink(storage_path) if File.exist?(storage_path)
     end
 
+    UUID_REGEXP = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i
+
+    def host_id=(host_id)
+      expect! host_id => UUID_REGEXP
+
+      return if @host_id == host_id
+
+      register_host(host_id: host_id)
+      @host_id = host_id
+    end
+
     def host_id
-      @host_id ||= atomic_set_and_get(storage_path) { register_host }
+      @host_id ||= atomic_set_and_get(storage_path) { register_host(host_id: nil) }
     end
 
     private
@@ -23,8 +39,12 @@ class Postjob::Host < Postjob::Record
     # the current user. We choose a tmp location for that reason. (A /var location
     # would be even better - however, our systems do not have a user-writable /var).
     def storage_path
-      env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
-      File.join Dir.tmpdir, "postjob.#{env}.#{Process.uid}.host_id"
+      @storage_path ||= begin
+        env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+        storage_path = File.join Dir.tmpdir, "postjob.#{env}.#{Process.uid}.host_id"
+        Postjob.logger.info "Keeping host identifier in #{storage_path}"
+        storage_path
+      end
     end
 
     def atomic_set_and_get(path)
@@ -43,10 +63,11 @@ class Postjob::Host < Postjob::Record
       value
     end
 
-    def register_host
+    def register_host(host_id:)
+      expect! host_id => [ nil, UUID_REGEXP ]
       attributes = {}
       Postjob.logger.debug "registering host w/#{attributes.inspect}"
-      ::Postjob::Queue.host_register(attributes)
+      ::Postjob::Queue.host_register(attributes, host_id: host_id)
     end
   end
 end

data/lib/postjob/migrations/001_helpers.sql

@@ -0,0 +1,19 @@
+CREATE OR REPLACE FUNCTION iff(cond BOOLEAN, value anyelement) RETURNS anyelement AS $$
+  BEGIN
+    IF cond THEN
+      RETURN value;
+    ELSE
+      RETURN NULL;
+    END IF;
+  END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION iff(cond BOOLEAN, value anyelement, else_value anyelement) RETURNS anyelement AS $$
+  BEGIN
+    IF cond THEN
+      RETURN value;
+    ELSE
+      RETURN else_value;
+    END IF;
+  END;
+$$ LANGUAGE plpgsql;

data/lib/postjob/migrations/007_job_results.sql

@@ -149,29 +149,3 @@ BEGIN
   PERFORM {SCHEMA_NAME}._wakeup_parent_job(p_worker_session_id, job_id);
 END;
 $$ LANGUAGE plpgsql;
-
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._set_job_zombie(
-  job_id BIGINT,
-  p_fast_mode BOOLEAN) RETURNS VOID AS $$
-DECLARE
-  p_worker_session_id UUID;
-BEGIN
-  p_worker_session_id := {SCHEMA_NAME}._null_uuid();
-
-  PERFORM {SCHEMA_NAME}._reset_job_processing(p_worker_session_id, job_id);
-
-  -- write error info
-  UPDATE {SCHEMA_NAME}.postjobs
-  SET
-    error='Zombie',
-    error_message='zombie',
-    error_backtrace=NULL,
-    failed_attempts=failed_attempts+1,
-    next_run_at=NULL
-  WHERE id=job_id;
-
-  -- prepare next run, if any
-  PERFORM {SCHEMA_NAME}._prepare_rerun(job_id, 'err', p_fast_mode);
-  PERFORM {SCHEMA_NAME}._wakeup_parent_job(p_worker_session_id, job_id);
-END;
-$$ LANGUAGE plpgsql;

data/lib/postjob/migrations/012_hosts.sql

@@ -1,5 +1,18 @@
 -- hosts ----------------------------------------------------------------------
 
+DO $$
+  BEGIN
+    CREATE TYPE {SCHEMA_NAME}.host_statuses AS ENUM (
+      'running',   -- host is running
+      'shutdown',  -- host is shutting down
+      'stopped'    -- host is not running
+    );
+  EXCEPTION
+    WHEN duplicate_object THEN RAISE DEBUG 'type {SCHEMA_NAME}.host_statuses already exists';
+  END;
+$$;
+
+
 -- The hosts table records available hosts. hosts are the basis for the
 -- sticky jobs feature - a sticky job is a job which can only be checked
 -- out by a specific host.
@@ -7,6 +20,10 @@
 
 CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.hosts (
   id UUID PRIMARY KEY  DEFAULT (gen_random_uuid()),     -- UUID identifying a worker **host**
+
+  -- The workflow status, one of  'running', 'shutdown', 'stopped'
+  status {SCHEMA_NAME}.host_statuses NOT NULL DEFAULT 'stopped',
+
   attributes JSONB NOT NULL DEFAULT '{}'::JSONB,
   created_at timestamp NOT NULL DEFAULT (now() at time zone 'utc')
 );
@@ -14,6 +31,14 @@ CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.hosts (
 CREATE INDEX IF NOT EXISTS hosts_attributes_idx
   ON {SCHEMA_NAME}.hosts USING GIN (attributes jsonb_path_ops);
 
+DO $$
+  BEGIN
+    ALTER TABLE {SCHEMA_NAME}.hosts ADD COLUMN status {SCHEMA_NAME}.host_statuses NOT NULL DEFAULT 'stopped';
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.hosts.host_statuses already exists';
+  END;
+$$;
+
 -- returns _null_host_id ------------------------------------------------------
 
 DO $$
@@ -28,15 +53,33 @@ $$ LANGUAGE plpgsql;
 
 -- host_register: registers a host --------------------------------------------
 
-CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.host_register(p_attrs JSONB)
+DROP FUNCTION IF EXISTS  {SCHEMA_NAME}.host_register(p_attrs JSONB);
+CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.host_register(p_attrs JSONB, v_id UUID)
   RETURNS UUID
 AS $$
-  DECLARE
-    v_id UUID;
   BEGIN
-    v_id := gen_random_uuid();
+    IF v_id IS NULL THEN
+      v_id := gen_random_uuid();
+    END IF;
+
     INSERT INTO {SCHEMA_NAME}.hosts (id, attributes)
-      VALUES (v_id, p_attrs);
+      VALUES (v_id, p_attrs)
+    ON CONFLICT(id) DO UPDATE SET attributes=p_attrs;
     RETURN v_id;
   END;
 $$ LANGUAGE plpgsql;
+
+
+-- wakeup runners after changing hosts
+
+-- when a host changes its status to shutdown, all of its runners should
+-- shutdown quickly.
+
+BEGIN;
+  DROP TRIGGER IF EXISTS _wakeup_runners ON {SCHEMA_NAME}.hosts;
+
+  CREATE TRIGGER _wakeup_runners AFTER UPDATE
+    ON {SCHEMA_NAME}.hosts
+    FOR EACH STATEMENT
+    EXECUTE PROCEDURE {SCHEMA_NAME}._wakeup_runners();
+COMMIT;

data/lib/postjob/migrations/013_worker_sessions.sql

@@ -24,6 +24,7 @@ CREATE TABLE IF NOT EXISTS {SCHEMA_NAME}.worker_sessions (
   host_id UUID NOT NULL REFERENCES {SCHEMA_NAME}.hosts ON DELETE CASCADE, -- UUID identifying a worker **host**
   client_socket VARCHAR,                                                  -- host:port of connection (from pg_stat_activity)
   workflows VARCHAR[] NOT NULL,                                           -- array of workflow versions available on that worker
+  queues VARCHAR[] NOT NULL,                                              -- array of queue names available on that worker
   attributes JSONB NOT NULL DEFAULT '{}'::JSONB,
   created_at timestamp NOT NULL DEFAULT (now() at time zone 'utc')
 );
@@ -38,7 +39,17 @@ DO $$
     null_uuid UUID := {SCHEMA_NAME}._null_uuid();
   BEGIN
     IF NOT EXISTS (SELECT 1 FROM {SCHEMA_NAME}.worker_sessions WHERE id = null_uuid) THEN
-      INSERT INTO {SCHEMA_NAME}.worker_sessions(id, host_id, workflows) VALUES(null_uuid, null_uuid, '{}');
+      INSERT INTO {SCHEMA_NAME}.worker_sessions(id, host_id, workflows, queues) VALUES(null_uuid, null_uuid, '{}', ARRAY[]::varchar[]);
     END IF;
   END;
 $$ LANGUAGE plpgsql;
+
+DO $$
+  BEGIN
+    ALTER TABLE {SCHEMA_NAME}.worker_sessions ADD COLUMN queues VARCHAR[];
+    UPDATE {SCHEMA_NAME}.worker_sessions SET queues = Array['ruby'] WHERE queues IS NULL;
+    ALTER TABLE {SCHEMA_NAME}.worker_sessions ALTER COLUMN queues SET NOT NULL;
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.worker_sessions.queues already exists';
+  END;
+$$;