refile-postgres 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3750ff6ae5e70ecd23b619f28d6a3c9a12150d09
-  data.tar.gz: ccaf3d9739e3b80e7c4ebbf76de3b124c33064ae
+  metadata.gz: b8bef59aba0412348cf839d73d050fe0884d0a76
+  data.tar.gz: 67e92c6a6f7ff603458ffa802c45af72468f2c4a
 SHA512:
-  metadata.gz: 2983f0299c9d796c8ba4a1b1831612b4bf2633ee410c7897aa4ad3da5b4f9b6acb1276fc576b8e7dd819504ab4b383fe00de5264b90b5df7339cf0398f0fc173
-  data.tar.gz: 16d74a9439671ae6ffd5a3970dad6b987619d56c8941bd8586673344d6ef44c33410cef0415aa5754b53014cf24d8d9b66f28dc59deb0be93589a2f20dae1e21
+  metadata.gz: fb11435d032ab11d7b3f7fe05ca49b2d9d50b3bce200d08d6d8b706b3b4d4c47f647c43483b10c3806424e8b62c051ed2d18c9b6ef8216d6c7fd8c4c56e683d1
+  data.tar.gz: ffce481c8f6aafd7c3a8665c496cab13d598b123ba662e06fd1b1bc169a999be377f3e32084ec970833cccbd864807fc73c889110ad2b715dc992a64893811f2

data/Dockerfile

@@ -0,0 +1,16 @@
+FROM ruby:2.3.0
+
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \
+    && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+    && apt-get update -qq \
+    && apt-get install -y build-essential libpq-dev postgresql-client-9.5 postgresql-contrib-9.5
+
+
+ENV APP_PATH=/app BUNDLE_JOBS=4 BUNDLE_RETRY=3 BUNDLE_PATH=/gems
+
+RUN mkdir ${APP_PATH}
+WORKDIR ${APP_PATH}
+
+ADD . ${APP_PATH}
+
+CMD bundle check || bundle install; bundle exec rspec spec

data/README.md

@@ -36,6 +36,13 @@ Or install it yourself as:
 
 ## Usage with Rails
 
+Application must have sql as schema format to properly dump OID type.
+
+   ```ruby
+   # config/application.rb
+   config.active_record.schema_format = :sql
+   ```
+
 Generate migration for table were to store list of attachments.
 
     $ rails g refile:postgres:migration
@@ -48,6 +55,11 @@ Generate initializer and set Refile::Postgres as `store` backend.
 
     $ rails g refile:postgres:initializer
 
+## Upgrade to 1.3.0
+
+If you have been using refile-postgres before 1.3.0 version then please follow [upgrade guide](https://github.com/krists/refile-postgres/blob/master/migration_to_1_3_0.md)
+
+
 ## Contributing
 
 1. Fork it ( https://github.com/krists/refile-postgres/fork )

data/docker-compose.yml

@@ -0,0 +1,22 @@
+app:
+  build: .
+  links:
+    - postgres
+  volumes_from:
+    - gems
+  environment:
+    POSTGRES_HOST: postgres
+    POSTGRES_PORT: 5432
+    POSTGRES_DB: refile_test
+    POSTGRES_USER: refile_postgres_test_user
+    POSTGRES_PASSWORD: refilepostgres
+postgres:
+  environment:
+    POSTGRES_USER: refile_postgres_test_user
+    POSTGRES_PASSWORD: refilepostgres
+    POSTGRES_DB: refile_test
+  image: postgres:9.5.1
+gems:
+  image: busybox
+  volumes:
+    - /gems

data/lib/generators/refile/postgres/migration/templates/migration.rb.erb

@@ -1,13 +1,33 @@
 class Create<%= table_name.camelize %> < ActiveRecord::Migration
   def up
-    create_table :<%= table_name %> do |t|
-      t.string :namespace, null: false
-    end
-    add_index :<%= table_name %>, :namespace
+    execute %Q{
+      CREATE TABLE <%= table_name %> (
+      id integer NOT NULL,
+      oid oid NOT NULL,
+      namespace character varying NOT NULL,
+      created_at timestamp without time zone DEFAULT ('now'::text)::timestamp without time zone
+      );
+
+      CREATE SEQUENCE <%= table_name %>_id_seq
+      START WITH 1
+      INCREMENT BY 1
+      NO MINVALUE
+      NO MAXVALUE
+      CACHE 1;
+
+      ALTER SEQUENCE <%= table_name %>_id_seq OWNED BY <%= table_name %>.id;
+
+      ALTER TABLE ONLY <%= table_name %> ALTER COLUMN id SET DEFAULT nextval('<%= table_name %>_id_seq'::regclass);
+
+      ALTER TABLE ONLY <%= table_name %> ADD CONSTRAINT <%= table_name %>_pkey PRIMARY KEY (id);
+
+      CREATE INDEX index_<%= table_name %>_on_namespace ON <%= table_name %> USING btree (namespace);
+
+      CREATE INDEX index_<%= table_name %>_on_oid ON <%= table_name %> USING btree (oid);
+    }
   end
 
   def drop
-    remove_index :<%= table_name %>, :namespace
     drop_table :<%= table_name %>
   end
 end

data/lib/refile/postgres/backend.rb

@@ -46,7 +46,7 @@ module Refile
                 connection.lo_write(handle, buffer)
               end
               uploadable.close
-              connection.exec_params("INSERT INTO #{registry_table} VALUES ($1::integer, $2::varchar);", [oid, namespace])
+              connection.exec_params("INSERT INTO #{registry_table} (oid, namespace) VALUES ($1::oid, $2::varchar);", [oid, namespace])
               Refile::File.new(self, oid.to_s)
             ensure
               connection.lo_close(handle)
@@ -80,9 +80,9 @@ module Refile
           connection.exec_params(%{
             SELECT count(*) FROM #{registry_table}
             INNER JOIN #{PG_LARGE_OBJECT_METADATA_TABLE}
-            ON #{registry_table}.id = #{PG_LARGE_OBJECT_METADATA_TABLE}.oid
+            ON #{registry_table}.oid = #{PG_LARGE_OBJECT_METADATA_TABLE}.oid
             WHERE #{registry_table}.namespace = $1::varchar
-            AND #{registry_table}.id = $2::integer;
+            AND #{registry_table}.oid = $2::integer;
           }, [namespace, id.to_s.to_i]) do |result|
             result[0]["count"].to_i > 0
           end
@@ -101,8 +101,14 @@ module Refile
         if exists?(id)
           with_connection do |connection|
             ensure_in_transaction(connection) do
-              connection.lo_unlink(id.to_s.to_i)
-              connection.exec_params("DELETE FROM #{registry_table} WHERE id = $1::integer;", [id])
+              rez = connection.exec_params(%{
+                SELECT * FROM #{registry_table}
+                WHERE #{registry_table}.oid = $1::integer
+                LIMIT 1
+              }, [id.to_s.to_i])
+              oid = rez[0]['oid'].to_i
+              connection.lo_unlink(oid)
+              connection.exec_params("DELETE FROM #{registry_table} WHERE oid = $1::integer;", [oid])
             end
           end
         end
@@ -114,12 +120,12 @@ module Refile
         with_connection do |connection|
           ensure_in_transaction(connection) do
             connection.exec_params(%{
-              SELECT * FROM #{registry_table}
-              INNER JOIN #{PG_LARGE_OBJECT_METADATA_TABLE} ON #{registry_table}.id = #{PG_LARGE_OBJECT_METADATA_TABLE}.oid
+              SELECT #{registry_table}.oid FROM #{registry_table}
+              INNER JOIN #{PG_LARGE_OBJECT_METADATA_TABLE} ON #{registry_table}.oid = #{PG_LARGE_OBJECT_METADATA_TABLE}.oid
               WHERE #{registry_table}.namespace = $1::varchar;
             }, [namespace]) do |result|
               result.each_row do |row|
-                connection.lo_unlink(row[0].to_s.to_i)
+                connection.lo_unlink(row[0].to_i)
               end
             end
             connection.exec_params("DELETE FROM #{registry_table} WHERE namespace = $1::varchar;", [namespace])

data/lib/refile/postgres/version.rb

@@ -1,5 +1,5 @@
 module Refile
   module Postgres
-    VERSION = "1.2.1"
+    VERSION = "1.3.0"
   end
 end

data/migration_to_1_3_0.md

@@ -0,0 +1,47 @@
+# Migration to refile-postgres version 1.3.0
+
+Please check [issue](https://github.com/krists/refile-postgres/issues/9) for more details.
+
+1) Change Rails schema dump format to SQL in your `config/application.rb` file
+   ```ruby
+   # Use structure.sql instead of schema.rb
+   config.active_record.schema_format = :sql
+   ```
+2) Change version number in Gemfile
+   ```ruby
+   gem 'refile-postgres', '~> 1.3.0'
+   ```
+3) Create Rails migration
+   ```
+   rails g migration refile_postgres_migration_to_1_3_0
+   ```
+4) Add content to migration
+   ```ruby
+   class RefilePostgresMigrationTo130 < ActiveRecord::Migration
+     def up
+       execute <<-SQL
+         DROP INDEX index_refile_attachments_on_namespace;
+         ALTER TABLE refile_attachments RENAME TO old_refile_attachments;
+         ALTER TABLE ONLY old_refile_attachments RENAME CONSTRAINT refile_attachments_pkey TO old_refile_attachments_pkey;
+         CREATE TABLE refile_attachments (
+           id integer NOT NULL,
+           oid oid NOT NULL,
+           namespace character varying NOT NULL,
+           created_at timestamp without time zone DEFAULT ('now'::text)::timestamp without time zone
+         );
+         ALTER TABLE ONLY refile_attachments ADD CONSTRAINT refile_attachments_pkey PRIMARY KEY (id);
+         ALTER SEQUENCE refile_attachments_id_seq RESTART OWNED BY refile_attachments.id;
+         ALTER TABLE ONLY refile_attachments ALTER COLUMN id SET DEFAULT nextval('refile_attachments_id_seq'::regclass);
+         INSERT INTO refile_attachments (oid, namespace) SELECT id, namespace FROM old_refile_attachments;
+         CREATE INDEX index_refile_attachments_on_namespace ON refile_attachments USING btree (namespace);
+         CREATE INDEX index_refile_attachments_on_oid ON refile_attachments USING btree (oid);
+         DROP TABLE old_refile_attachments;
+       SQL
+     end
+
+     def down
+       raise ActiveRecord::IrreversibleMigration
+     end
+   end
+   ```
+5) Now it is safe to run [vacuumlo](http://www.postgresql.org/docs/9.5/static/vacuumlo.html)

data/spec/refile/postgres/backend_spec.rb

@@ -8,7 +8,7 @@ describe Refile::Postgres::Backend do
     context "when not using procs and providing PG::Connection directly" do
       let(:connection_or_proc) { connection }
       it "reuses the same PG::Connection" do
-        expect(backend.with_connection { |c| c.db }).to eq("refile_test")
+        expect(backend.with_connection { |c| c.db }).to eq(TEST_DB_NAME)
       end
     end
 
@@ -25,7 +25,7 @@ describe Refile::Postgres::Backend do
       context "when lambda does yield a PG::Connection" do
         let(:connection_or_proc) { lambda { |&blk| blk.call(connection) } }
         it "is usable in queries" do
-          expect(backend.with_connection { |c| c.db }).to eq("refile_test")
+          expect(backend.with_connection { |c| c.db }).to eq(TEST_DB_NAME)
         end
       end
     end
@@ -64,6 +64,18 @@ describe Refile::Postgres::Backend do
     end
   end
 
+  describe "Orphaned large object cleaning" do
+    let(:connection_or_proc) { test_connection }
+    let(:backend) { Refile::Postgres::Backend.new(connection_or_proc, max_size: 10000 ) }
+    it "does not garbage collect attachments after vacuumlo call" do
+      uploadable = File.open(File.expand_path(__FILE__))
+      file = backend.upload(uploadable)
+      expect(backend.exists?(file.id)).to eq(true)
+      run_vacuumlo
+      expect(backend.exists?(file.id)).to eq(true)
+    end
+  end
+
   context "Refile Provided tests" do
     let(:connection_or_proc) { connection }
     it_behaves_like :backend

data/spec/spec_helper.rb

@@ -8,30 +8,59 @@ require "refile/spec_helper"
 require "pg"
 require "pry"
 require "refile/postgres"
+require "open3"
 
 WebMock.disable!(:except => [:codeclimate_test_reporter])
 
+TEST_DB_NAME = ENV.fetch('POSTGRES_DB', 'refile_test')
+TEST_DB_HOST = ENV.fetch('POSTGRES_HOST','localhost')
+TEST_DB_USER = ENV.fetch('POSTGRES_USER','refile_postgres_test_user')
+TEST_DB_PASSWD = ENV.fetch('POSTGRES_PASSWORD','refilepostgres')
 module DatabaseHelpers
   def test_connection
-    @@connection ||= PG.connect(host: 'localhost', dbname: 'refile_test', user: 'refile_postgres_test_user', password: 'refilepostgres')
+    @@connection ||= PG.connect(host: TEST_DB_HOST, dbname: TEST_DB_NAME, user: TEST_DB_USER, password: TEST_DB_PASSWD)
   end
 
-  def create_registy_table
-    test_connection.exec %{
-      CREATE TABLE IF NOT EXISTS #{Refile::Postgres::Backend::DEFAULT_REGISTRY_TABLE}
-      (
-        id serial NOT NULL,
-        namespace character varying(255),
-        CONSTRAINT refile_backend_lo_oids_pkey PRIMARY KEY (id)
-      )
-      WITH(
-        OIDS=FALSE
+  def create_registy_table(name = Refile::Postgres::Backend::DEFAULT_REGISTRY_TABLE)
+    test_connection.exec %Q{
+      DROP TABLE IF EXISTS #{name};
+      CREATE TABLE #{name} (
+      id integer NOT NULL,
+      oid oid NOT NULL,
+      namespace character varying NOT NULL,
+      created_at timestamp without time zone DEFAULT ('now'::text)::timestamp without time zone
       );
+
+      CREATE SEQUENCE #{name}_id_seq
+      START WITH 1
+      INCREMENT BY 1
+      NO MINVALUE
+      NO MAXVALUE
+      CACHE 1;
+
+      ALTER SEQUENCE #{name}_id_seq OWNED BY #{name}.id;
+
+      ALTER TABLE ONLY #{name} ALTER COLUMN id SET DEFAULT nextval('#{name}_id_seq'::regclass);
+
+      ALTER TABLE ONLY #{name} ADD CONSTRAINT #{name}_pkey PRIMARY KEY (id);
+
+      CREATE INDEX index_#{name}_on_namespace ON #{name} USING btree (namespace);
+
+      CREATE INDEX index_#{name}_on_oid ON #{name} USING btree (oid);
     }
   end
 
-  def drop_registry_table
-    test_connection.exec %{ DROP TABLE IF EXISTS #{Refile::Postgres::Backend::DEFAULT_REGISTRY_TABLE} CASCADE; }
+  def drop_registry_table(name = Refile::Postgres::Backend::DEFAULT_REGISTRY_TABLE)
+    test_connection.exec %{ DROP TABLE IF EXISTS #{name} CASCADE; }
+  end
+
+  def run_vacuumlo
+    command = "export PGPASSWORD=#{TEST_DB_PASSWD}; vacuumlo  -h #{TEST_DB_HOST} -U #{TEST_DB_USER} -w -v #{TEST_DB_NAME}"
+    Open3.popen3(command) do |stdin, stdout, stderr, thread|
+      stdin.close
+      IO.copy_stream(stderr, $stderr)
+      IO.copy_stream(stdout, $stdout)
+    end
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: refile-postgres
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Krists Ozols
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-16 00:00:00.000000000 Z
+date: 2016-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: refile
@@ -160,20 +160,12 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
-- Berksfile
-- Berksfile.lock
+- Dockerfile
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
-- Vagrantfile
-- cookbooks/dev-essential/metadata.rb
-- cookbooks/dev-essential/recipes/default.rb
-- cookbooks/postgresql/metadata.rb
-- cookbooks/postgresql/recipes/default.rb
-- cookbooks/ruby/attributes/default.rb
-- cookbooks/ruby/metadata.rb
-- cookbooks/ruby/recipes/default.rb
+- docker-compose.yml
 - lib/generators/refile/postgres/initializer/USAGE
 - lib/generators/refile/postgres/initializer/initializer_generator.rb
 - lib/generators/refile/postgres/initializer/templates/refile.rb
@@ -185,6 +177,7 @@ files:
 - lib/refile/postgres/backend/reader.rb
 - lib/refile/postgres/smart_transaction.rb
 - lib/refile/postgres/version.rb
+- migration_to_1_3_0.md
 - postgres-setup
 - refile-postgres.gemspec
 - spec/refile/postgres/backend_spec.rb

data/Berksfile

@@ -1,4 +0,0 @@
-source "https://supermarket.chef.io"
-
-cookbook 'apt', '~> 2.7.0'
-cookbook 'locale', '~> 1.0.2'

data/Berksfile.lock

@@ -1,7 +0,0 @@
-DEPENDENCIES
-  apt (~> 2.7.0)
-  locale (~> 1.0.2)
-
-GRAPH
-  apt (2.7.0)
-  locale (1.0.2)

data/Vagrantfile

@@ -1,24 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-Vagrant.configure(2) do |config|
-  config.vm.box = "chef/ubuntu-14.04"
-
-  config.ssh.forward_agent = true
-  config.vm.synced_folder ".", "/home/vagrant/project"
-
-  config.vm.provider "virtualbox" do |vb|
-    vb.memory = "2048"
-    vb.cpus = 2
-  end
-
-  config.berkshelf.enabled = false
-
-  config.vm.provision "chef_solo" do |chef|
-    chef.cookbooks_path = ["cookbooks", "berks-cookbooks"]
-    chef.log_level = :info
-    chef.add_recipe "dev-essential"
-    chef.add_recipe "ruby"
-    chef.add_recipe "postgresql"
-  end
-end

data/cookbooks/dev-essential/metadata.rb

@@ -1,8 +0,0 @@
-name             'dev-essential'
-maintainer       'Krists Ozols'
-maintainer_email 'krists.ozols@gmail.com'
-license          'MIT'
-description      'Installs/Configures some essentials for development'
-version          '0.1.0'
-
-depends 'locale', '~> 1.0.2'

data/cookbooks/dev-essential/recipes/default.rb

@@ -1,9 +0,0 @@
-include_recipe "locale"
-
-%w{
-  git-core curl vim tmux
-}.each do |p|
-  package p do
-    action :install
-  end
-end

data/cookbooks/postgresql/metadata.rb

@@ -1,8 +0,0 @@
-name             'postgresql'
-maintainer       'Krists Ozols'
-maintainer_email 'krists.ozols@gmail.com'
-license          'MIT'
-description      'Installs/Configures PostgreSQL 9.4'
-version          '0.1.0'
-
-depends 'apt', '~> 2.7.0'

data/cookbooks/postgresql/recipes/default.rb

@@ -1,29 +0,0 @@
-apt_repository 'postgresql' do
-  uri          'http://apt.postgresql.org/pub/repos/apt/'
-  distribution 'trusty-pgdg'
-  components   ['main']
-  key          'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
-  action :add
-end
-
-%w{
-  postgresql-9.4
-  postgresql-client-9.4
-  postgresql-contrib-9.4
-  postgresql-server-dev-9.4
-}.each do |p|
-  package p do
-    action :install
-  end
-end
-
-execute 'Create vagrant postgresql user' do
-  guard = <<-EOH
-    psql -U postgres -c "select * from pg_user where
-    usename='vagrant'" |
-    grep -c vagrant
-  EOH
-  user 'postgres'
-  command %{psql -U postgres -c "CREATE ROLE vagrant PASSWORD 'vagrant' SUPERUSER CREATEDB LOGIN;"}
-  not_if guard, user: 'postgres'
-end

data/cookbooks/ruby/attributes/default.rb

@@ -1,2 +0,0 @@
-default[:ruby_install][:version] = '0.5.0'
-default[:chruby][:version] = '0.3.9'

data/cookbooks/ruby/metadata.rb

@@ -1,6 +0,0 @@
-name             'ruby'
-maintainer       'Krists Ozols'
-maintainer_email 'krists.ozols@gmail.com'
-license          'MIT'
-description      'Installs/Configures ruby-installer and chruby'
-version          '0.1.0'

data/cookbooks/ruby/recipes/default.rb

@@ -1,52 +0,0 @@
-
-remote_file "/tmp/ruby-install-#{node[:ruby_install][:version]}.tar.gz" do
-  source "https://github.com/postmodern/ruby-install/archive/v#{node[:ruby_install][:version]}.tar.gz"
-  action :create
-end
-
-bash "Extract ruby-install source code" do
-  code <<-EOH
-    cd /tmp
-    tar -zxvf ruby-install-#{node[:ruby_install][:version]}.tar.gz
-  EOH
-  not_if { ::File.exists?("/tmp/ruby-install-#{node[:ruby_install][:version]}") }
-end
-
-execute 'Install ruby-install' do
-  cwd "/tmp/ruby-install-#{node[:ruby_install][:version]}"
-  command 'make install'
-  action :run
-  not_if { ::File.exists?("/usr/local/bin/ruby-install") }
-end
-
-remote_file "/tmp/chruby-#{node[:chruby][:version]}.tar.gz" do
-  source "https://github.com/postmodern/chruby/archive/v#{node[:chruby][:version]}.tar.gz"
-  action :create
-end
-
-bash "Extract chruby source code" do
-  code <<-EOH
-    cd /tmp
-    tar -zxvf chruby-#{node[:chruby][:version]}.tar.gz
-  EOH
-  not_if { ::File.exists?("/tmp/chruby-#{node[:chruby][:version]}") }
-end
-
-execute 'Install chruby' do
-  cwd "/tmp/chruby-#{node[:chruby][:version]}"
-  command 'make install'
-  action :run
-  not_if { ::File.exists?("/usr/local/bin/chruby") }
-end
-
-file "/etc/profile.d/chruby.sh" do
-  owner 'root'
-  group 'root'
-  mode '0644'
-  content <<-EOC
-    if [ -n "$BASH_VERSION" ] || [ -n "$ZSH_VERSION" ]; then
-      source /usr/local/share/chruby/chruby.sh
-      source /usr/local/share/chruby/auto.sh
-    fi
-  EOC
-end