que 0.14.3 → 1.0.0.beta

data/lib/que/metajob.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# A thin wrapper around a job's data that lets us do things like sort easily and
+# make sure that run_at is in the format we want.
+
+module Que
+  class Metajob
+    SORT_KEYS = [:priority, :run_at, :id].freeze
+
+    attr_reader :job
+
+    def initialize(job)
+      set_job(job)
+    end
+
+    def set_job(job)
+      if (run_at = job.fetch(:run_at)).is_a?(Time)
+        job[:run_at] = run_at.utc.iso8601(6)
+      end
+
+      @job = job
+    end
+
+    def id
+      job.fetch(:id)
+    end
+
+    def <=>(other)
+      k1 = job
+      k2 = other.job
+
+      SORT_KEYS.each do |key|
+        value1 = k1.fetch(key)
+        value2 = k2.fetch(key)
+
+        return -1 if value1 < value2
+        return  1 if value1 > value2
+      end
+
+      0
+    end
+
+    def priority_sufficient?(threshold)
+      threshold.nil? || job.fetch(:priority) <= threshold
+    end
+  end
+end

data/lib/que/migrations.rb

@@ -3,28 +3,33 @@
 module Que
   module Migrations
     # In order to ship a schema change, add the relevant up and down sql files
-    # to the migrations directory, and bump the version both here and in the
-    # add_que generator template.
-    CURRENT_VERSION = 3
+    # to the migrations directory, and bump the version here.
+    CURRENT_VERSION = 4
 
     class << self
-      def migrate!(options = {:version => CURRENT_VERSION})
+      def migrate!(version:)
         Que.transaction do
-          version = options[:version]
+          current = db_version
 
-          if (current = db_version) == version
+          if current == version
             return
           elsif current < version
-            direction = 'up'
+            direction = :up
             steps = ((current + 1)..version).to_a
           elsif current > version
-            direction = 'down'
+            direction = :down
             steps = ((version + 1)..current).to_a.reverse
           end
 
           steps.each do |step|
-            sql = File.read("#{File.dirname(__FILE__)}/migrations/#{step}/#{direction}.sql")
-            Que.execute(sql)
+            filename = [
+              File.dirname(__FILE__),
+              'migrations',
+              step,
+              direction,
+            ].join('/') << '.sql'
+
+            Que.execute(File.read(filename))
           end
 
           set_db_version(version)
@@ -32,18 +37,20 @@ module Que
       end
 
       def db_version
-        result = Que.execute <<-SQL
-          SELECT relname, description
-          FROM pg_class
-          LEFT JOIN pg_description ON pg_description.objoid = pg_class.oid
-          WHERE relname = 'que_jobs'
-        SQL
+        result =
+          Que.execute <<-SQL
+            SELECT relname, description
+            FROM pg_class
+            LEFT JOIN pg_description ON pg_description.objoid = pg_class.oid
+            WHERE relname = 'que_jobs'
+          SQL
 
         if result.none?
           # No table in the database at all.
           0
         elsif (d = result.first[:description]).nil?
-          # There's a table, it was just created before the migration system existed.
+          # There's a table, it was just created before the migration system
+          # existed.
           1
         else
           d.to_i

data/lib/que/migrations/4/down.sql

@@ -0,0 +1,48 @@
+ALTER TABLE que_jobs RESET (fillfactor);
+
+ALTER TABLE que_jobs DROP CONSTRAINT que_jobs_pkey;
+DROP INDEX que_poll_idx;
+DROP INDEX que_jobs_data_gin_idx;
+
+DROP TRIGGER que_job_notify ON que_jobs;
+DROP FUNCTION que_job_notify();
+DROP TRIGGER que_state_notify ON que_jobs;
+DROP FUNCTION que_state_notify();
+DROP FUNCTION que_determine_job_state(que_jobs);
+DROP TABLE que_lockers;
+
+DROP TABLE que_values;
+DROP INDEX que_jobs_args_gin_idx;
+
+ALTER TABLE que_jobs RENAME COLUMN id TO job_id;
+ALTER SEQUENCE que_jobs_id_seq RENAME TO que_jobs_job_id_seq;
+
+ALTER TABLE que_jobs RENAME COLUMN last_error_message TO last_error;
+
+DELETE FROM que_jobs WHERE (finished_at IS NOT NULL OR expired_at IS NOT NULL);
+
+ALTER TABLE que_jobs
+  DROP CONSTRAINT error_length,
+  DROP CONSTRAINT queue_length,
+  DROP CONSTRAINT job_class_length,
+  DROP CONSTRAINT valid_args,
+  DROP COLUMN finished_at,
+  DROP COLUMN expired_at,
+  ALTER args TYPE JSON using args::json;
+
+UPDATE que_jobs
+SET
+  queue = CASE queue WHEN 'default' THEN '' ELSE queue END,
+  last_error = last_error || coalesce(E'\n' || last_error_backtrace, '');
+
+ALTER TABLE que_jobs
+  DROP COLUMN data,
+  DROP COLUMN last_error_backtrace,
+  ALTER COLUMN args SET NOT NULL,
+  ALTER COLUMN args SET DEFAULT '[]',
+  ALTER COLUMN queue SET DEFAULT '';
+
+ALTER TABLE que_jobs
+  ADD PRIMARY KEY (queue, priority, run_at, job_id);
+
+DROP FUNCTION que_validate_tags(jsonb);

data/lib/que/migrations/4/up.sql

@@ -0,0 +1,265 @@
+ALTER TABLE que_jobs SET (fillfactor = 90);
+ALTER TABLE que_jobs RENAME COLUMN last_error TO last_error_message;
+ALTER TABLE que_jobs RENAME COLUMN job_id TO id;
+ALTER TABLE que_jobs RENAME COLUMN args TO old_args;
+ALTER SEQUENCE que_jobs_job_id_seq RENAME TO que_jobs_id_seq;
+
+ALTER TABLE que_jobs
+  ADD COLUMN last_error_backtrace text,
+  ADD COLUMN finished_at timestamptz,
+  ADD COLUMN expired_at timestamptz,
+  ADD COLUMN args JSONB,
+  ADD COLUMN data JSONB;
+
+ALTER TABLE que_jobs DROP CONSTRAINT que_jobs_pkey;
+
+UPDATE que_jobs
+SET
+  queue = CASE queue WHEN '' THEN 'default' ELSE queue END,
+  last_error_backtrace =
+    CASE
+    WHEN last_error_message ~ '\n'
+      THEN left(regexp_replace(last_error_message, '^[^\n]+\n', ''), 10000)
+    ELSE
+      NULL
+    END,
+  last_error_message = left(substring(last_error_message from '^[^\n]+'), 500),
+  args =
+    CASE json_typeof(old_args)
+    WHEN 'array' THEN old_args::jsonb
+    ELSE jsonb_build_array(old_args)
+    END,
+  data = '{}'::jsonb;
+
+CREATE FUNCTION que_validate_tags(tags_array jsonb) RETURNS boolean AS $$
+  SELECT bool_and(
+    jsonb_typeof(value) = 'string'
+    AND
+    char_length(value::text) <= 100
+  )
+  FROM jsonb_array_elements(tags_array)
+$$
+LANGUAGE SQL;
+
+-- Now that we're done rewriting data, add new indexes.
+CREATE INDEX que_poll_idx ON que_jobs (queue, priority, run_at, id) WHERE (finished_at IS NULL AND expired_at IS NULL);
+CREATE INDEX que_jobs_data_gin_idx ON que_jobs USING gin (data jsonb_path_ops);
+CREATE INDEX que_jobs_args_gin_idx ON que_jobs USING gin (args jsonb_path_ops);
+
+ALTER TABLE que_jobs
+  ADD PRIMARY KEY (id),
+  DROP COLUMN old_args,
+  ALTER COLUMN queue SET DEFAULT 'default',
+  ALTER COLUMN args SET DEFAULT '[]',
+  ALTER COLUMN args SET NOT NULL,
+  ALTER COLUMN data SET DEFAULT '{}',
+  ALTER COLUMN data SET NOT NULL,
+  ADD CONSTRAINT queue_length CHECK (
+    char_length(queue) <= 100
+  ),
+  ADD CONSTRAINT job_class_length CHECK (
+    char_length(
+      CASE job_class
+      WHEN 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper' THEN
+        args->0->>'job_class'
+      ELSE
+        job_class
+      END
+    ) <= 200
+  ),
+  ADD CONSTRAINT valid_args CHECK (
+    (jsonb_typeof(args) = 'array')
+  ),
+  ADD CONSTRAINT valid_data CHECK (
+    (jsonb_typeof(data) = 'object')
+    AND
+    (
+      (NOT data ? 'tags')
+      OR
+      (
+        (jsonb_typeof(data->'tags') = 'array')
+        AND
+        (jsonb_array_length(data->'tags') <= 5)
+        AND
+        (public.que_validate_tags(data->'tags'))
+      )
+    )
+  ),
+  ADD CONSTRAINT error_length CHECK (
+    (char_length(last_error_message) <= 500) AND
+    (char_length(last_error_backtrace) <= 10000)
+  );
+
+-- This is somewhat heretical, but we're going to need some more flexible
+-- storage to support various features without requiring a ton of migrations,
+-- which would be a lot of hassle for users. Hopefully this will be used smartly
+-- and sparingly (famous last words).
+CREATE TABLE que_values (
+  key text PRIMARY KEY,
+  value jsonb NOT NULL DEFAULT '{}',
+  CONSTRAINT valid_value CHECK (jsonb_typeof(value) = 'object')
+)
+WITH (FILLFACTOR=90);
+
+CREATE UNLOGGED TABLE que_lockers (
+  pid               integer NOT NULL CONSTRAINT que_lockers_pkey PRIMARY KEY,
+  worker_count      integer NOT NULL,
+  worker_priorities integer[] NOT NULL,
+  ruby_pid          integer NOT NULL,
+  ruby_hostname     text    NOT NULL,
+  queues            text[]  NOT NULL,
+  listening         boolean NOT NULL,
+
+  CONSTRAINT valid_worker_priorities CHECK (
+    (array_ndims(worker_priorities) = 1)
+    AND
+    (array_length(worker_priorities, 1) IS NOT NULL) -- Doesn't do zero.
+  ),
+
+  CONSTRAINT valid_queues CHECK (
+    (array_ndims(queues) = 1)
+    AND
+    (array_length(queues, 1) IS NOT NULL) -- Doesn't do zero.
+  )
+);
+
+CREATE FUNCTION que_job_notify() RETURNS trigger AS $$
+  DECLARE
+    locker_pid integer;
+    sort_key json;
+  BEGIN
+    -- Don't do anything if the job is scheduled for a future time.
+    IF NEW.run_at IS NOT NULL AND NEW.run_at > now() THEN
+      RETURN null;
+    END IF;
+
+    -- Pick a locker to notify of the job's insertion, weighted by their number
+    -- of workers. Should bounce pseudorandomly between lockers on each
+    -- invocation, hence the md5-ordering, but still touch each one equally,
+    -- hence the modulo using the job_id.
+    SELECT pid
+    INTO locker_pid
+    FROM (
+      SELECT *, last_value(row_number) OVER () + 1 AS count
+      FROM (
+        SELECT *, row_number() OVER () - 1 AS row_number
+        FROM (
+          SELECT *
+          FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id
+          WHERE listening AND queues @> ARRAY[NEW.queue]
+          ORDER BY md5(pid::text || id::text)
+        ) t1
+      ) t2
+    ) t3
+    WHERE NEW.id % count = row_number;
+
+    IF locker_pid IS NOT NULL THEN
+      -- There's a size limit to what can be broadcast via LISTEN/NOTIFY, so
+      -- rather than throw errors when someone enqueues a big job, just
+      -- broadcast the most pertinent information, and let the locker query for
+      -- the record after it's taken the lock. The worker will have to hit the
+      -- DB in order to make sure the job is still visible anyway.
+      SELECT row_to_json(t)
+      INTO sort_key
+      FROM (
+        SELECT
+          'job_available' AS message_type,
+          NEW.queue       AS queue,
+          NEW.priority    AS priority,
+          NEW.id          AS id,
+          -- Make sure we output timestamps as UTC ISO 8601
+          to_char(NEW.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at
+      ) t;
+
+      PERFORM pg_notify('que_listener_' || locker_pid::text, sort_key::text);
+    END IF;
+
+    RETURN null;
+  END
+$$
+LANGUAGE plpgsql;
+
+CREATE TRIGGER que_job_notify
+  AFTER INSERT ON que_jobs
+  FOR EACH ROW
+  EXECUTE PROCEDURE public.que_job_notify();
+
+CREATE FUNCTION que_determine_job_state(job public.que_jobs) RETURNS text AS $$
+  SELECT
+    CASE
+    WHEN job.expired_at  IS NOT NULL    THEN 'expired'
+    WHEN job.finished_at IS NOT NULL    THEN 'finished'
+    WHEN job.error_count > 0            THEN 'errored'
+    WHEN job.run_at > CURRENT_TIMESTAMP THEN 'scheduled'
+    ELSE                                     'ready'
+    END
+$$
+LANGUAGE SQL;
+
+CREATE FUNCTION que_state_notify() RETURNS trigger AS $$
+  DECLARE
+    row record;
+    message json;
+    previous_state text;
+    current_state text;
+  BEGIN
+    IF TG_OP = 'INSERT' THEN
+      previous_state := 'nonexistent';
+      current_state  := public.que_determine_job_state(NEW);
+      row            := NEW;
+    ELSIF TG_OP = 'DELETE' THEN
+      previous_state := public.que_determine_job_state(OLD);
+      current_state  := 'nonexistent';
+      row            := OLD;
+    ELSIF TG_OP = 'UPDATE' THEN
+      previous_state := public.que_determine_job_state(OLD);
+      current_state  := public.que_determine_job_state(NEW);
+
+      -- If the state didn't change, short-circuit.
+      IF previous_state = current_state THEN
+        RETURN null;
+      END IF;
+
+      row := NEW;
+    ELSE
+      RAISE EXCEPTION 'Unrecognized TG_OP: %', TG_OP;
+    END IF;
+
+    SELECT row_to_json(t)
+    INTO message
+    FROM (
+      SELECT
+        'job_change' AS message_type,
+        row.id       AS id,
+        row.queue    AS queue,
+
+        coalesce(row.data->'tags', '[]'::jsonb) AS tags,
+
+        to_char(row.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at,
+        to_char(now()      AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS time,
+
+        CASE row.job_class
+        WHEN 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper' THEN
+          coalesce(
+            row.args->0->>'job_class',
+            'ActiveJob::QueueAdapters::QueAdapter::JobWrapper'
+          )
+        ELSE
+          row.job_class
+        END AS job_class,
+
+        previous_state AS previous_state,
+        current_state  AS current_state
+    ) t;
+
+    PERFORM pg_notify('que_state', message::text);
+
+    RETURN null;
+  END
+$$
+LANGUAGE plpgsql;
+
+CREATE TRIGGER que_state_notify
+  AFTER INSERT OR UPDATE OR DELETE ON que_jobs
+  FOR EACH ROW
+  EXECUTE PROCEDURE public.que_state_notify();

data/lib/que/poller.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+module Que
+  class Poller
+    # The following SQL statement locks a batch of jobs using a Postgres
+    # recursive CTE [1].
+    #
+    # As noted by the Postgres documentation, it may be slightly easier to
+    # think about this expression as iteration rather than recursion, despite
+    # the `RECURSION` nomenclature defined by the SQL standards committee.
+    # Recursion is used here so that jobs in the table can be iterated one-by-
+    # one until a lock can be acquired, where a non-recursive `SELECT` would
+    # have the undesirable side-effect of locking jobs unnecessarily. For
+    # example, the following might lock more than five jobs during execution:
+    #
+    #   SELECT (j).*, pg_try_advisory_lock((j).id) AS locked
+    #   FROM public.que_jobs AS j
+    #   LIMIT 5;
+    #
+    # The CTE will initially produce an "anchor" from the non-recursive term
+    # (i.e. before the `UNION`), and then use it as the contents of the
+    # working table as it continues to iterate through `que_jobs` looking for
+    # locks. The jobs table has an index on (priority, run_at, id) which
+    # allows it to walk the jobs table in a stable manner. As noted above, the
+    # recursion examines/locks one job at a time. Every time the recursive
+    # entry runs, it's output becomes the new contents of the working table,
+    # and what was previously in the working table is appended to the final
+    # result set. For more information on the basic workings of recursive
+    # CTEs, see http://www.postgresql.org/docs/devel/static/queries-with.html
+    #
+    # The polling query is provided a JSONB hash of desired job priorities.
+    # For example, if the locker has three workers free that can work a
+    # priority less than 5, and two workers free that can work a priority less
+    # than 10, the provided priority document is `{"5":3,"10":2}`. The query
+    # uses this information to decide what jobs to lock - if only high-
+    # priority workers were available, it wouldn't make sense to retrieve low-
+    # priority jobs.
+    #
+    # As each job is retrieved from the table, it is passed to
+    # lock_and_update_priorities() (which, for future flexibility, we define
+    # as a temporary function attached to the connection rather than embedding
+    # permanently into the DB schema). lock_and_update_priorities() attempts
+    # to lock the given job and, if it is able to, updates the priorities
+    # document to reflect that a job was available for that given priority.
+    # When the priorities document is emptied (all the counts of desired jobs
+    # for the various priorities have reached zero and been removed), the
+    # recursive query returns an empty result and the recursion breaks. This
+    # also happens if there aren't enough appropriate jobs in the jobs table.
+    #
+    # Also note the use of JOIN LATERAL to combine the job data with the
+    # output of lock_and_update_priorities(). The naive approach would be to
+    # write the SELECT as `SELECT (j).*, (lock_and_update_priorities(..., j)).*`,
+    # but the asterisk-expansion of the latter composite row causes the function
+    # to be evaluated twice, and to thereby take the advisory lock twice,
+    # which complicates the later unlocking step.
+    #
+    # Thanks to RhodiumToad in #postgresql for help with the original
+    # (simpler) version of the recursive job lock CTE.
+
+    SQL[:poll_jobs] =
+      %{
+        WITH RECURSIVE jobs AS (
+          SELECT
+            (j).*,
+            l.locked,
+            l.remaining_priorities
+          FROM (
+            SELECT j
+            FROM public.que_jobs AS j
+            WHERE queue = $1::text
+              AND NOT id = ANY($2::bigint[])
+              AND priority <= pg_temp.que_highest_remaining_priority($3::jsonb)
+              AND run_at <= now()
+              AND finished_at IS NULL AND expired_at IS NULL
+            ORDER BY priority, run_at, id
+            LIMIT 1
+          ) AS t1
+          JOIN LATERAL (SELECT * FROM pg_temp.lock_and_update_priorities($3::jsonb, j)) AS l ON true
+          UNION ALL (
+            SELECT
+              (j).*,
+              l.locked,
+              l.remaining_priorities
+            FROM (
+              SELECT
+                remaining_priorities,
+                (
+                  SELECT j
+                  FROM public.que_jobs AS j
+                  WHERE queue = $1::text
+                    AND NOT id = ANY($2::bigint[])
+                    AND priority <= pg_temp.que_highest_remaining_priority(jobs.remaining_priorities)
+                    AND run_at <= now()
+                    AND finished_at IS NULL AND expired_at IS NULL
+                    AND (priority, run_at, id) >
+                      (jobs.priority, jobs.run_at, jobs.id)
+                  ORDER BY priority, run_at, id
+                  LIMIT 1
+                ) AS j
+
+              FROM jobs
+              WHERE jobs.id IS NOT NULL AND jobs.remaining_priorities != '{}'::jsonb
+              LIMIT 1
+            ) AS t1
+            JOIN LATERAL (SELECT * FROM pg_temp.lock_and_update_priorities(remaining_priorities, j)) AS l ON true
+          )
+        )
+        SELECT *
+        FROM jobs
+        WHERE locked
+      }
+
+    attr_reader \
+      :connection,
+      :queue,
+      :poll_interval,
+      :last_polled_at,
+      :last_poll_satisfied
+
+    def initialize(
+      connection:,
+      queue:,
+      poll_interval:
+    )
+      @connection          = connection
+      @queue               = queue
+      @poll_interval       = poll_interval
+      @last_polled_at      = nil
+      @last_poll_satisfied = nil
+
+      Que.internal_log :poller_instantiate, self do
+        {
+          backend_pid:   connection.backend_pid,
+          queue:         queue,
+          poll_interval: poll_interval,
+        }
+      end
+    end
+
+    def poll(
+      priorities:,
+      held_locks:
+    )
+
+      return unless should_poll?
+
+      expected_count = priorities.inject(0){|s,(p,c)| s + c}
+
+      jobs =
+        connection.execute_prepared(
+          :poll_jobs,
+          [
+            @queue,
+            "{#{held_locks.to_a.join(',')}}",
+            JSON.dump(priorities),
+          ]
+        )
+
+      @last_polled_at      = Time.now
+      @last_poll_satisfied = expected_count == jobs.count
+
+      Que.internal_log :poller_polled, self do
+        {
+          queue:        @queue,
+          locked:       jobs.count,
+          priorities:   priorities,
+          held_locks:   held_locks.to_a,
+          newly_locked: jobs.map { |key| key.fetch(:id) },
+        }
+      end
+
+      jobs.map! { |job| Metajob.new(job) }
+    end
+
+    def should_poll?
+      # Never polled before?
+      last_poll_satisfied.nil? ||
+      # Plenty of jobs were available last time?
+      last_poll_satisfied == true ||
+      poll_interval_elapsed?
+    end
+
+    def poll_interval_elapsed?
+      return unless interval = poll_interval
+      (Time.now - last_polled_at) > interval
+    end
+
+    class << self
+      # Manage some temporary infrastructure (specific to the connection) that
+      # we'll use for polling. These could easily be created permanently in a
+      # migration, but that'd require another migration if we wanted to tweak
+      # them later.
+
+      def setup(connection)
+        connection.execute <<-SQL
+          -- Temporary composite type we need for our queries to work.
+          CREATE TYPE pg_temp.que_query_result AS (
+            locked boolean,
+            remaining_priorities jsonb
+          );
+
+          CREATE FUNCTION pg_temp.lock_and_update_priorities(priorities jsonb, job que_jobs)
+          RETURNS pg_temp.que_query_result
+          AS $$
+            WITH
+              -- Take the lock in a CTE because we want to use the result
+              -- multiple times while only taking the lock once.
+              lock_taken AS (
+                SELECT pg_try_advisory_lock((job).id) AS taken
+              ),
+              relevant AS (
+                SELECT priority, count
+                FROM (
+                  SELECT
+                    key::smallint AS priority,
+                    value::text::integer AS count
+                  FROM jsonb_each(priorities)
+                  ) t1
+                WHERE priority >= (job).priority
+                ORDER BY priority ASC
+                LIMIT 1
+              )
+            SELECT
+              (SELECT taken FROM lock_taken), -- R
+              CASE (SELECT taken FROM lock_taken)
+              WHEN false THEN
+                -- Simple case - we couldn't lock the job, so don't update the
+                -- priorities hash.
+                priorities
+              WHEN true THEN
+                CASE count
+                WHEN 1 THEN
+                  -- Remove the priority from the JSONB doc entirely, rather
+                  -- than leaving a zero entry in it.
+                  priorities - priority::text
+                ELSE
+                  -- Decrement the value in the JSONB doc.
+                  jsonb_set(
+                    priorities,
+                    ARRAY[priority::text],
+                    to_jsonb(count - 1)
+                  )
+                END
+              END
+            FROM relevant
+          $$
+          STABLE
+          LANGUAGE SQL;
+
+          CREATE FUNCTION pg_temp.que_highest_remaining_priority(priorities jsonb) RETURNS smallint AS $$
+            SELECT max(key::smallint) FROM jsonb_each(priorities)
+          $$
+          STABLE
+          LANGUAGE SQL;
+        SQL
+      end
+
+      def cleanup(connection)
+        connection.execute <<-SQL
+          DROP FUNCTION pg_temp.que_highest_remaining_priority(jsonb);
+          DROP FUNCTION pg_temp.lock_and_update_priorities(jsonb, que_jobs);
+          DROP TYPE pg_temp.que_query_result;
+        SQL
+      end
+    end
+  end
+end