bricolage-streamingload 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 41d99ce0dce7affca77b305ce527c5ec53e733b1
-  data.tar.gz: bd2254dab228018edf50debcf4627deef3bbdbd0
+  metadata.gz: 6fcd9200426dc3c2389566e0b6c5618d7ee2ae2f
+  data.tar.gz: 6880ed950da15b2550c706bb0525364286889e23
 SHA512:
-  metadata.gz: 66855f5839f86deb3c6af0500090bed718bd6860b50a83e6a11f1c79e822b81b3d7888ff5417391f4f1b0a669242ed5de3d9a9f4c3fe4e500d88ad77acfbe7fa
-  data.tar.gz: c18b88f15e4cd75095f5cf72863921a43ee467bb1b03f5b3481bf63a09e1a0e319f3afcd4e79e5902bdb6f62695528edd962ed561c29bfc5f6dc173ed029eb89
+  metadata.gz: e278f2dbf3e9cc26b3c1420bea5a4e580af7eaf006b01e2bfecd641efa553214583b3291160d80911c9b7ed06ee00164eb76a64b13a4ef7ab586dcfaae790de7
+  data.tar.gz: ea493c3e3f2ee8644c3615dc1667fe2902120c8af5c86bfe71b882020aa967801c70b12eff69c0032f6ee486781fcde475a7c0e18e9d73bdb84ec2a32c62c535

data/lib/bricolage/streamingload/objectbuffer.rb

@@ -274,16 +274,27 @@ module Bricolage
                     , obj.object_id
                     , load_batch_size
                 from
+                    -- unassigned objects
                     (
                         select
                             data_source_id
-                            , object_url
-                            , min(object_id) as object_id
+                            , uniq_objects.object_url
+                            , object_id
                         from
-                            strload_objects
-                        group by
-                            1, 2
-                    ) obj
+                            (
+                                select
+                                    min(object_id) as object_id
+                                    , object_url
+                                from
+                                    strload_objects
+                                group by
+                                    object_url
+                            ) uniq_objects
+                            inner join strload_objects using(object_id)
+                            left outer join strload_task_objects using(object_id)
+                        where
+                            task_id is null
+                     ) obj
 
                     -- tasks without objects
                     inner join (
@@ -297,16 +308,11 @@ module Bricolage
                             using (schema_name, table_name)
                         where
                             -- unassigned objects
-                            task_id not in (select task_id from strload_task_objects)
+                            task_id not in (select distinct task_id from strload_task_objects)
                         group by
                             1
                     ) task
                     using (data_source_id)
-
-                    left outer join strload_task_objects task_obj
-                    using (object_id)
-                where
-                    task_obj.object_id is null   -- unassigned to a task
                 ) as t
             where
                 object_count <= load_batch_size   -- limit number of objects assigned to single task

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.5.0'
+    VERSION = '0.5.1'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Minero Aoki